爬去实习网站的相关数据

# -*- coding: utf-8 -*-
"""
Created on Tue Jan 22 19:31:16 2019

@author: Administrator
"""

import pandas as pd
import requests
from lxml import etree
import chardet
import re

def shixiwang(url):
    
    url = url
    header = {"User-Agent": 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36'}
    
    rqg  = requests.get(url,headers = header)
    rqg.encoding = chardet.detect(rqg.content)['encoding']
    html = etree.HTML(rqg.text)
    
    
    
    #取出岗位名
    J_name = html.xpath('//div/div[@class="job-pannel-one"]/dl/dt/a/text()')
    name  = [re.sub('[\t\n\r ]','',J_name[i]) for i in range(len(J_name))]
#    print(len(name))
    
    #取出公司名
    C_name = html.xpath('//dd/div[@class="company-info-title"]/a/text()')
#    print(len(C_name))
    
    #取出学历
    education = html.xpath('//dd/span[@class="job-educational"]/text()')
#    print(len(education))
    
    C_address = html.xpath('//div/div[@class="company-info"]/span/a/text()')
#    print(len(C_address))
    
    #取出发布日期
    date = html.xpath('//div/span[@class="job-time"]/text()')
#    print(len(date))
    
    
    #取出月薪
    silver = html.xpath('//div/div[@class="company-info-des"]/text()')
    silver = [re.sub('[\t\n\r ]','',silver[i]) for i in range(len(silver))]
#    print(len(silver))
    
    data = {'J_name' :name,
            'C_name':C_name,
            'silver':silver,
            'education':education,
            'C_address':C_address,
            'date':date
            }
    
    data = pd.DataFrame(data)
    return data 



    


res1 = pd.DataFrame(columns = ['J_name','C_name','silver','education','C_address','date'])
url = []



for i in range(1,8):
   url.append('https://www.shixi.com/search/index?key=爬虫&page='+str(i))
    
for i in range(0,len(url)):
    res = shixiwang(url[i])
    res1 = pd.concat([res1,res],axis=0)

部分结果