爬虫学习04.
对加密数据的爬取
import requests
from lxml import etree
import base64
import os
from urllib import request
url = 'http://jandan.net/ooxx/page-46'
headers = {
'User-Agent':'ozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
}
page_text = requests.get(url=url, headers=headers).text
#print(page_text)
tree = etree.HTML(page_text)
code_list = tree.xpath('//div[@class="text"]/p/span[@class="img-hash"]/text()')
#print(code_list)
if not os.path.exists('dandan'):
os.mkdir('dandan')
for code in code_list:
#print(code)
src = base64.b64decode(code).decode()
img_url = 'https:' + src
filepath = 'dandan/' + src.split('/')[-1]
#print(src)
request.urlretrieve(url=img_url, filename=filepath)
print(filepath + '下载成功')
二.验证码处理及模拟登陆
1.相关的门户网站在进行登录的时候，如果用户连续登录的次数超过3次或者5次的时候，就会在登录页中动态生成验证码。通过验证码达到分流和反爬的效果。
2.- 1.对携带验证码的页面数据进行抓取
- 2.可以将页面数据中验证码进行解析，验证码图片下载到本地
- 3.可以将验证码图片提交给三方平台进行识别，返回验证码图片上的数据值
- 云打码平台：
- 1.在官网中进行注册（普通用户和开发者用户）
- 2.登录开发者用户：
- 1.实例代码的下载（开发文档-》调用实例及最新的DLL-》PythonHTTP实例下载）
- 2.创建一个软件：我的软件-》添加新的软件
-3.使用示例代码中的源码文件中的代码进行修改，让其识别验证码图片中的数据值
3.账号:1355144989@qq.com 密码1355144989
模拟登陆爬取人人网
import requests
from lxml import etree
from urllib import request
url = 'http://www.renren.com/SysHome.do'
headers = {
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
}
page_text = requests.get(url=url, headers=headers).text
tree = etree.HTML(page_text)
src = tree.xpath('//*[@id="verifyPic_login"]/@src')[0]
print(src)
img = request.urlretrieve(url=src, filename='./renren.jpg')
# 登录
code = get_code(2004,'./renren.jpg')
session = requests.Session()
login_url = 'http://www.renren.com/ajaxLogin/login?1=1&uniqueTimestamp=201903160368'
data = {
"email":"15516092050",
"icode":code,
"origURL":"http://www.renren.com/home",
"domain":"renren.com",
"key_id":'1',
"captcha_type":"web_login",
"password":"5e088a2ee22d34dd081aac25578e67bd3a2d851cdfbcf1f0c9ab7056bd1bad62",
"rkey":"3f4696f6fa1b89e9061868300bf11484",
"f":"http%3A%2F%2Fwww.renren.com%2F969395731",
}
login_page = session.post(url=login_url, headers=headers,data=data)
print(login_page.headers)
detail_url = 'http://www.renren.com/969395731'
detail_content = session.get(url=url,headers=headers).text
with open('./renren.html', 'w', encoding='utf-8') as f:
f.write(detail_content)
print('下载成功')
三.并发爬取视频
from multiprocessing.dummy import Pool # 开启多线程
from lxml import etree
import requests
import re
pool = Pool(5) # 创建一个线程池
url = 'https://www.pearvideo.com/category_3'
headers = {
'User-Agent':'ozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
}
page_text = requests.get(url=url, headers = headers).text
tree = etree.HTML(page_text)
video_url_list = tree.xpath('//*[@id="listvideoListUl"]/li/div/a/@href')
print(video_url_list)
after_url_list = []
film_name_list = []
for video_url in video_url_list:
detail_url = 'https://www.pearvideo.com/' + video_url
video_detail_page = requests.get(url = detail_url, headers=headers).text
after_url = re.findall('ldUrl="",srcUrl="(.*?)"', video_detail_page)[0]
trees = etree.HTML(video_detail_page)
film_name = trees.xpath('//div[@class="video-main"]/div/img[@class="img"]/@alt')[0]
film_name_list.append(film_name)
after_url_list.append(after_url)
print(after_url_list)
get_video_data = lambda after_url:requests.get(url=after_url, headers=headers).content
video_data_list = pool.map(get_video_data,after_url_list )
def get_video_name(video_data):
name = str(video_data_list.index(video_data))
with open(f'./{name}.mp4', 'wb') as f:
f.write(video_data)
print(name + '下载成功')
pool.map(get_video_name,video_data_list )
