py爬虫实战
一、糗事百科视频爬取
先找到对应的页面，分析视频的来源，通过正则匹配到链接，然后再通过“美味的汤”来获取对应的视频的标题，进行下载
import requests import re from bs4 import BeautifulSoup url="https://www.qiushibaike.com/video/" headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36"} resp=requests.get(url,headers=headers) content = resp.content.decode('utf-8') soup=BeautifulSoup(content,'html.parser') text=resp.text # print(text) list=re.findall(r'<source src="(.*)" type=\'video/mp4\' />',text) divs=soup.find_all('div',class_="content") title=[] for item in divs: title.append(item.find('span').text.strip()) lst=[] for item in list: lst.append("https:"+item) count=0 for item in lst: resp=requests.get(item,headers=headers) with open("video/"+str(title[count])+".mp4","wb") as file: file.write(resp.content) print("已下载完第"+str(count)+"个") count+=1 print("视频下载完毕")
二、链家二手房数据
本次主要利用“BeautifulSoup”和mysql数据库
import re import requests import mysql.connector from bs4 import BeautifulSoup class LianJia(): mydb=mysql.connector.connect(host='localhost',user='root',password='fengge666',database='hotel') mycursor=mydb.cursor() #初始化 def __init__(self): self.url="https://bj.lianjia.com/chengjiao/pg{0}/" self.headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'} def send_request(self,url): resp=requests.get(url,headers=self.headers) if resp: self.parse_content(resp) def parse_content(self,resp): lst=[] html=resp.text bs=BeautifulSoup(html,'html.parser') ul=bs.find('ul',class_='listContent') li_lst=ul.find_all('li') for item in li_lst: title=item.find('div',class_='title').text houseInfo=item.find('div',class_='houseInfo').text data=item.find('div',class_='dealDate').text money=item.find('div',class_='totalPrice').text flood=item.find('div',class_='positionInfo').text price=item.find('div',class_='unitPrice').text span=item.find('span',class_='dealCycleTxt') span_lst=span.find_all('span') agent=item.find('a',class_='agent_name').text lst.append((title,houseInfo,data,money,flood,price,span_lst[0].text,span_lst[1].text,agent)) #开始存储到数据库 self.write_mysql(lst) def write_mysql(self,lst): # print(self.mydb) tuple_lst=tuple(lst) sql="insert into house (title,houseInfo,data,money,flood,price,current_money,current_data,agent) values (%s,%s,%s,%s,%s,%s,%s,%s,%s)" self.mycursor.executemany(sql,tuple_lst) self.mydb.commit() def start(self): for i in range(1,11): full_url=self.url.format(i) resp=self.send_request(full_url) if resp: self.parse_content(self,resp) if __name__ == '__main__': lianjia=LianJia() lianjia.start()
三、爬取招聘职位网信息
主要利用beautifulSoup对其网页进行分析，去除对应的数据，将其保存在xlsx文件里，对其进行分页爬取
import re import openpyxl from bs4 import BeautifulSoup import requests import time def send_request(id,page): url='https://www.jobui.com/company/{0}/jobs/p{1}'.format(id,page) headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36"} resp=requests.get(url,headers=headers) return resp.text lst=[] def parse_html(html): bs=BeautifulSoup(html,'html.parser') job_lst=bs.find_all('div',class_='c-job-list') for item in job_lst: name=item.find('h3').text div_tag=item.find('div',class_='job-desc') span_tag=div_tag.find_all('span') url=item.find('a',class_='job-name')['href'] url='https://www.jobui.com'+url lst.append([name,span_tag[0].text,span_tag[1].text,url]) def save(lst): wk=openpyxl.Workbook() sheet=wk.active for item in lst: sheet.append(item) wk.save('招聘信息.xlsx') def start(id,pages): for page in range(1,pages+1): resp_data=send_request(id,page) parse_html(resp_data) time.sleep(2) save(lst) if __name__ == '__main__': id='10375749' pages=3 start(id,pages)
四、爬取QQ音乐排行旁信息
这个排行榜的信息封装在json数据里，因此需要我们对json格式进行分析，取出想要的数据，保存到数据库里
import requests import re import mysql.connector def get_request(): url="https://u.y.qq.com/cgi-bin/musics.fcg?-=getUCGI9057052357882678&g_tk=130572444&sign=zzan8er9xsqr1dg0y3e7df30d14b15a2b335cedcd0d6c6c883f&loginUin=1751520702&hostUin=0&format=json&inCharset=utf8&outCharset=utf-8¬ice=0&platform=yqq.json&needNewCode=0&data=%7B%22detail%22%3A%7B%22module%22%3A%22musicToplist.ToplistInfoServer%22%2C%22method%22%3A%22GetDetail%22%2C%22param%22%3A%7B%22topId%22%3A4%2C%22offset%22%3A0%2C%22num%22%3A20%2C%22period%22%3A%222020-07-23%22%7D%7D%2C%22comm%22%3A%7B%22ct%22%3A24%2C%22cv%22%3A0%7D%7D" resp=requests.get(url) return resp.json() def parse_data(): data=[] data_json=get_request() lst_song=data_json['detail']['data']['data']['song'] for item in lst_song: data.append((item['rank'],item['title'],item['singerName'])) return data def save(): mydb=mysql.connector.connect(host='localhost',user='root',password='fengge666',database='python_database') mycursor=mydb.cursor() sql='insert into song_table values(%s,%s,%s)' lst=parse_data() mycursor.executemany(sql,lst) mydb.commit() print(mycursor.rowcount,'记录插入成功') if __name__ == '__main__': save()
五、爬取12306车次信息
在这里也是对json数据进行解析，唯一有点难度的就是城市的名称和其对应的简称之间的对应，找到对应的城市与简称的json数据报，解析后，得到对应的字典。
然后再对起分析对应的数据，存储到相应的xlsx文件里。
最终的程序模式是：输入起始站，终点站以及出发时间，就会自动帮你找对应的车次以及相应的信息
import requests import re import openpyxl def send_request(begin,end,data): lst=getHcity() begin=lst[begin] end=lst[end] url='https://kyfw.12306.cn/otn/leftTicket/query?leftTicketDTO.train_date={0}&leftTicketDTO.from_station={1}&leftTicketDTO.to_station={2}&purpose_codes=ADULT'.format(data,begin,end) headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36' ,'Cookie':'_uab_collina=159514803739447239524678; JSESSIONID=C6B40F3602421EA3F1C176AAAD6D07CD; tk=t4DWOVEUjJCURbdg5YymB_o_T5iBATU7sT2cagOk978qrx1x0; RAIL_EXPIRATION=1595441480862; RAIL_DEVICEID=dCwOxY9htFoUbg-W-ZiiJivIJneE0K0MYpVRFCEGJZVXr2VWjywrVdOvEJ6HKFapkeqFwD82pjGtJv0fB1SeILpr-60WLkdvjz6zV-hcnclaYrz1AcbOskdjaz3e3fJd007cLRkk4OiauQxiu6zjnhVnI4fytM01; BIGipServerpool_passport=283968010.50215.0000; route=6f50b51faa11b987e576cdb301e545c4; _jc_save_fromStation=%u90AF%u90F8%2CHDP; _jc_save_toStation=%u77F3%u5BB6%u5E84%2CSJP; _jc_save_fromDate=2020-07-19; _jc_save_toDate=2020-07-19; _jc_save_wfdc_flag=dc; BIGipServerotn=3990290698.50210.0000; BIGipServerpassport=887619850.50215.0000; uKey=97a13289be6445586b819425f91b9bcbcc15446c5f37bceb8352cc085d1017a4; current_captcha_type=Z'} resp=requests.get(url,headers=headers) resp.encoding='utf-8' # print(resp.text) return resp def pare_json(resp,city): json_ticket=resp.json() data_lst=json_ticket['data']['result'] content=data_lst[0].split('|') #遍历车次信息 lst=[] lst.append(['车次','起始站','到达站','一等座','二等座','软卧','硬卧','硬座','出行日期','出发时间','到达时间']) for item in data_lst: d=item.split('|') # d[3]为车次 # d[6]查询起始站 # d[7]查询到达站 # d[31]一等座 # d[30]二等座 # d[29]硬座 # d[23]软卧 # d[28]硬卧 # d[8] 出发时间 # d[9] 到达时间 # d[13]出行时间 lst.append([d[3],city[d[6]],city[d[7]],d[31],d[30],d[23],d[28],d[29],d[13],d[8],d[9]]) return lst def start(begin,end,data): lst=pare_json(send_request(begin,end,data),getcity()) wk = openpyxl.Workbook() sheet = wk.active for item in lst: sheet.append(item) wk.save('车票查询.xlsx') def getcity(): url="https://kyfw.12306.cn/otn/resources/js/framework/station_name.js?station_version=1.9151" headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'} resp=requests.get(url,headers=headers) resp.encoding='utf-8' stations=re.findall('([\u4e00-\u9fa5]+)\|([A-Z]+)',resp.text) stations_data=dict(stations) stations_d={} for item in stations_data: stations_d[stations_data[item]]=item return stations_d def getHcity(): url = "https://kyfw.12306.cn/otn/resources/js/framework/station_name.js?station_version=1.9151" headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'} resp = requests.get(url, headers=headers) resp.encoding = 'utf-8' stations = re.findall('([\u4e00-\u9fa5]+)\|([A-Z]+)', resp.text) stations_data = dict(stations) return stations_data if __name__ == '__main__': begin=input('begin:') end=input('end:') data=input('data:') start(begin,end,data)
六、笔趣阁小说爬取
闲来无事，回忆当初看过的奇幻小说，想来就盘他吧！！！
import re import requests from bs4 import BeautifulSoup import demjson import pymysql import os headers = {'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36'}#创建头部信息 url='https://www.xsbiquge.com/12_12735/' ht='https://www.xsbiquge.com' r=requests.get(url,headers=headers) content=r.content.decode('utf-8') soup = BeautifulSoup(content, 'html.parser') if not os.path.exists('校园纯情霸主小说'): os.mkdir('校园纯情霸主小说') content=soup.find('div',id='list') Title = str(content.dt.text[:-2]).strip() print(Title) #len(content.dl.contents)-1 lst_dd=content.find_all('dd') for item in lst_dd: link=ht+item.a['href'] name=item.a.text r = requests.get(link, headers=headers) content = r.content.decode('utf-8') soup = BeautifulSoup(content, 'html.parser') text=soup.find('div',id='content').text.replace(' ','\n') with open('校园纯情霸主小说' + '/' + name + '.txt', 'a',encoding='utf-8')as f: f.write(text) print("已完成"+name+" 的爬取！！！")
七、毒物特征与症状提取
这是和老师正在做的一个项目，需要将毒物的特征描述文字与对应的一些症状进行匹对，然后生成一个表，又该症状则为1，反之则为0
import csv import re import pandas as pd import openpyxl import xlrd lst_xlsx=['二级其他检查.xlsx','二级库呼吸.xlsx','二级库尿液检查.xlsx','二级库循环.xlsx','二级库泌尿生殖.xlsx','二级库消化.xlsx','二级库甲状腺.xlsx','二级库皮肤.xlsx','二级库眼部.xlsx','二级库神经.xlsx', '二级库粪便检查.xlsx','二级库肌电图.xlsx','二级库肝功能肾功能.xlsx','二级库胃部检查.xlsx','二级库脑电图.xlsx','二级库血液检查.xlsx','二级心电图.xlsx','其它.xlsx'] name_lst = [] content_lst = [] ans_lst=[] def get_dw(): wk = xlrd.open_workbook('duwuxiangqing.xls') table = wk.sheet_by_name('Sheet1') rows = table.nrows cols = table.ncols for i in range(2, rows): name = table.cell(i, 2).value str = table.cell(i, 13).value + table.cell(i, 14).value + table.cell(i, 17).value + table.cell(i, 18).value content_lst.append(str) name_lst.append(name) # for i in range(len(content_lst)): # print(name_lst[i],content_lst[i]) def get_title(): lst_title=[] lst_title.append("Name") for item in lst_xlsx: wk = openpyxl.load_workbook(item) sheet = wk.active rows = sheet.max_row cols = sheet.max_column for i in range(1, cols + 1): for j in range(1, rows + 1): size = sheet.cell(j, i).value if (size != None): lst_title.append(size) return lst_title def head(): global flag, num lst_title=[] lst_title.append("Name") #构建毒物名字 for item in name_lst: k=[] k.append(item) ans_lst.append(k) #遍历毒物特征 p=0 for cont_item in content_lst: cont_item=str(cont_item) print("目前正在进行第："+str(p)) #读取二级库里的特征 for item in lst_xlsx: wk = openpyxl.load_workbook(item) sheet = wk.active rows = sheet.max_row cols = sheet.max_column for i in range(1, cols + 1): flag=0 num=0 for j in range(1, rows + 1): size = sheet.cell(j, i).value if(size!=None): num+=1 if cont_item.find(size)>0 : flag=1 if flag==1: for f in range(num): ans_lst[p].append("1") else: for f in range(num): ans_lst[p].append("0") p=p+1 lst_title=get_title() write_title(lst_title,ans_lst) def write_title(title,ans): f = open('data.csv', 'w',encoding='utf-8',newline='') # 2. 基于文件对象构建 csv写入对象 csv_writer = csv.writer(f) # 3. 构建列表头 csv_writer.writerow(title) for item in ans: csv_writer.writerow(item) if __name__ == '__main__': get_dw() head()
data.csv部分截图：
