四.爬取拉勾网实例

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Author : shenqiang
import  csv
import  requests
def getHeaders():
    headers={
        'Content-Type':'application/x-www-form-urlencoded; charset=UTF-8',
        'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
        'Cookie':'_ga=GA1.2.1237290736.1534169036; user_trace_token=20180813220356-b7e42516-9f01-11e8-bb78-525400f775ce;
 LGUID=20180813220356-b7e428ad-9f01-11e8-bb78-525400f775ce; index_location_city=%E5%85%A8%E5%9B%BD; _gid=GA1.2.675811712.1540794503; JSESSIONID=ABAAABAAAGFABEF93F47251563A52306423D37E945D2C54; _gat=1; 
LGSID=20181029213144-fa3c8e13-db7e-11e8-b51c-525400f775ce; PRE_UTM=; PRE_HOST=www.bing.com; PRE_SITE=https%3A%2F%2Fwww.bing.com%2F; PRE_LAND=https%3A%2F%2Fwww.lagou.com%2F; Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1539529521,1539785285,1540794503,1540819902;
 SEARCH_ID=ae3ae41a58d94802a68e848d36c30711; Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1540819909; 
LGRID=20181029213151-fe7324dc-db7e-11e8-b51c-525400f775ce',
'Referer':'https://www.lagou.com/jobs/list_%E8%87%AA%E5%8A%A8%E5%8C%96%E6%B5%8B%E8%AF%95%E5%B7%
A5%E7%A8%8B%E5%B8%88?labelWords=sug&fromSearch=true&suginput
=%E8%87%AA%E5%8A%A8%E5%8C%96%E6%B5%8B%E8%AF%95'}
    return headers
def laGou(url='https://www.lagou.com/jobs/positionAjax.json?needAddtionalResult=false',page=2):
    positions = []
    r = requests.post(
        url=url,
        headers=getHeaders(),
        data={'first': False, 'pn': page, 'kd': '自动化测试工程师'})
    for i in range(15):
        city = r.json()['content']['positionResult']['result'][i]['city']
        education = r.json()['content']['positionResult']['result'][i]['education']
        workYear = r.json()['content']['positionResult']['result'][i]['workYear']
        positionAdvantage = r.json()['content']['positionResult']['result'][i]['positionAdvantage']
        salary = r.json()['content']['positionResult']['result'][i]['salary']
        companyFullName = r.json()['content']['positionResult']['result'][i]['companyFullName']
        positionLables = r.json()['content']['positionResult']['result'][i]['positionLables']
        position = {
            '公司名称': companyFullName,
            '城市': city,
            '学历': education,
            '工作年限': workYear,
            '薪资': salary,
            '工作标签': positionLables,
            '福利': positionAdvantage
        }
        positions.append(position)
    for item in positions:
        print(item)
if __name__ == '__main__':
    for item in range(1, 31):
        laGou(page=item)