爬取招聘网信息

一、爬取boss直聘网

import pandas as pd
import requests
from lxml import etree
import chardet
import re
 

def boss_Recruitment_network(url) :   
    url = url
    header = {"User-Agent": 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36'}

    rqg  = requests.get(url,headers = header)
    rqg.encoding = chardet.detect(rqg.content)['encoding']
    html = etree.HTML(rqg.text)

    name = html.xpath('//div/div/h3/a/div[@class="job-title"]/text()')
    salary = html.xpath('//div/div/h3/a/span[@class="red"]/text()')
    address = html.xpath('//div/div[@class="info-primary"]/p/text()')
    company =html.xpath('//div/div/div/h3[@class="name"]/a/text()')
    work_type = html.xpath('//div/div/div[@class="company-text"]/p/text()')

    work_e=[]
    education_b=[]
    address1 =[]  
    C_name =[]  
    name1 =[]   
    salary1 =[]   
    work_type1 =[] 


    for i in range(len(name)):
        if re.findall('数据',name[i]) and re.findall('经验',address[i*3+1]) :
            work_e.append(address[i*3+1])
            education_b.append(address[i*3+2]) 
            address1.append(address[i*3] )
            C_name.append(company[i])
            name1.append(name[i])
            salary1.append(salary[i])
            work_type1.append(work_type[3*i] )

    s = {
        'C_name' :C_name,
        'W_name':name1,
        'salary':salary1,
        'education':education_b,
        'work_e':work_e,
        'address':address1,
        'work_type':work_type1
        }

    s = pd.DataFrame(s)
    
    return s

t=pd.DataFrame()
for i in range(20):
    url = 'https://www.zhipin.com/c101280100/?query=%E5%AE%9E%E4%B9%A0%E7%94%9F&page='+str(i)+'&ka=page-'+str(i)
    print(url)
    y = boss_Recruitment_network(url)
    t=  pd.concat([t,y])  
t