scrapy
scrapy
scrapy是个什么鬼？
- 帮我们提供一个可扩展功能齐全的爬虫框架。
安装：
Linux/mac
- pip3 install scrapy
Windows:
- 安装twsited
a. pip3 install wheel
b. 下载twisted http://www.lfd.uci.edu/~gohlke/pythonlibs/#twisted
c. 进入下载目录，执行 pip3 install Twisted-xxxxx.whl
- 安装scrapy
d. pip3 install scrapy -i http://pypi.douban.com/simple --trusted-host pypi.douban.com
- 安装pywin32
e. pip3 install pywin32 -i http://pypi.douban.com/simple --trusted-host pypi.douban.com
快速使用：
Scrapy：
创建project:
scrapy startproject xianglong
cd xianglong
scrapy genspider chouti chouti.com
写代码
scrapy crawl chouti --nolog
scrapy相关：
1. spider，编写爬虫程序，去解析并处理请求。
def parse():
- HtmlXPathSelector
- yield item
- yield request
2. item/pipelines
配置：
ITEM_PIPELINES = {
'xianglong.pipelines.XianglongPipeline': 300,
}
使用：
class XianglongPipeline(object):
def process_item(self, item, spider):
self.f.write(item['href']+'\n')
self.f.flush()
return item
def open_spider(self, spider):
"""
爬虫开始执行时，调用
:param spider:
:return:
"""
self.f = open('url.log','w')
def close_spider(self, spider):
"""
爬虫关闭时，被调用
:param spider:
:return:
"""
self.f.close()
中间件
下载中间件
问题：scrapy中如何添加代理？
解决方案：
方式一：内置添加代理功能
# -- coding: utf-8 --
import os
import scrapy
from scrapy.http import Request
class ChoutiSpider(scrapy.Spider):
name = 'chouti'
allowed_domains = ['chouti.com']
start_urls = ['https://dig.chouti.com/']
def start_requests(self):
os.environ['HTTP_PROXY'] = "http://192.168.11.11"
# 写死，不能随机变动
for url in self.start_urls:
yield Request(url=url,callback=self.parse)
def parse(self, response):
print(response)
方式二：自定义下载中间件
import random
import base64
import six
def to_bytes(text, encoding=None, errors='strict'):
"""Return the binary representation of text. If text
is already a bytes object, return it as-is.
#为了兼容py2，py2字符串类型就是bytes类型，py3字符串为unicode字符
"""
if isinstance(text, bytes):
return text
if not isinstance(text, six.string_types):
raise TypeError('to_bytes must receive a unicode, str or bytes '
'object, got %s' % type(text).name)
if encoding is None:
encoding = 'utf-8'
return text.encode(encoding, errors)
class MyProxyDownloaderMiddleware(object):
def process_request(self, request, spider):
proxy_list = [
{'ip_port': '111.11.228.75:80', 'user_pass': 'xxx:123'},
{'ip_port': '120.198.243.22:80', 'user_pass': ''},
{'ip_port': '111.8.60.9:8123', 'user_pass': ''},
{'ip_port': '101.71.27.120:80', 'user_pass': ''},
{'ip_port': '122.96.59.104:80', 'user_pass': ''},
{'ip_port': '122.224.249.122:8088', 'user_pass': ''},
]
proxy = random.choice(proxy_list)
if proxy['user_pass'] is not None:
request.meta['proxy'] = to_bytes("http://%s" % proxy['ip_port'])
encoded_user_pass = base64.encodestring(to_bytes(proxy['user_pass']))
request.headers['Proxy-Authorization'] = to_bytes('Basic ' + encoded_user_pass)
else:
request.meta['proxy'] = to_bytes("http://%s" % proxy['ip_port'])
配置：
DOWNLOADER_MIDDLEWARES = {
# 'xiaohan.middlewares.MyProxyDownloaderMiddleware': 543,
}
问题：scrapy中如何处理https
掏钱：
pass
不掏钱：
from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory
from twisted.internet.ssl import (optionsForClientTLS, CertificateOptions, PrivateCertificate)
class MySSLFactory(ScrapyClientContextFactory):
def getCertificateOptions(self):
from OpenSSL import crypto
v1 = crypto.load_privatekey(crypto.FILETYPE_PEM, open('/client.key.unsecure', mode='r').read())
v2 = crypto.load_certificate(crypto.FILETYPE_PEM, open('/client.pem', mode='r').read())
return CertificateOptions(
privateKey=v1, # pKey对象
certificate=v2, # X509对象
verify=False,
method=getattr(self, 'method', getattr(self, '_ssl_method', None))
)
配置：
DOWNLOADER_HTTPCLIENTFACTORY = "scrapy.core.downloader.webclient.ScrapyHTTPClientFactory"
DOWNLOADER_CLIENTCONTEXTFACTORY = "xiaohan.middlewares.MySSLFactory"
总结：
问：下载中间件的作用？
答：在每次下载前和下载后对请求和响应可以定制功能。例如：user-agent/代理/cookie
爬虫中间件
编写：
middlewares.py
class XiaohanSpiderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the spider middleware does not modify the
# passed objects.
def init(self):
pass
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
return s
每次下载完成之后，未执行parse函数之前。
def process_spider_input(self, response, spider):
# Called for each response that goes through the spider
# middleware and into the spider.
Should return None or raise an exception.
print('process_spider_input',response)
return None
def process_spider_output(self, response, result, spider):
# Called with the results returned from the Spider, after
# it has processed the response.
Must return an iterable of Request, dict or Item objects.
print('process_spider_output',response)
for i in result:
yield i
def process_spider_exception(self, response, exception, spider):
# Called when a spider or process_spider_input() method
# (from other spider middleware) raises an exception.
Should return either None or an iterable of Response, dict
or Item objects.
pass
爬虫启动时，第一次执行start_requests时，触发。（只执行一次）
def process_start_requests(self, start_requests, spider):
# Called with the start requests of the spider, and works
# similarly to the process_spider_output() method, except
# that it doesn’t have a response associated.
Must return only requests (not items).
print('process_start_requests')
for r in start_requests:
yield r
应用：
SPIDER_MIDDLEWARES = {
'xiaohan.middlewares.XiaohanSpiderMiddleware': 543,
}
扩展+信号
单纯扩展：
extends.py
class MyExtension(object):
def init(self):
pass
@classmethod
def from_crawler(cls, crawler):
obj = cls()
return obj
配置：
EXTENSIONS = {
'xiaohan.extends.MyExtension':500,
}
扩展+信号：
extends.py
from scrapy import signals
class MyExtension(object):
def init(self):
pass
@classmethod
def from_crawler(cls, crawler):
obj = cls()
# 在爬虫打开时，触发spider_opened信号相关的所有函数：xxxxxxxxxxx
crawler.signals.connect(obj.xxxxxxxxxxx1, signal=signals.spider_opened)
# 在爬虫关闭时，触发spider_closed信号相关的所有函数：xxxxxxxxxxx
crawler.signals.connect(obj.uuuuuuuuuu, signal=signals.spider_closed)
return obj
def xxxxxxxxxxx1(self, spider):
print('open')
def uuuuuuuuuu(self, spider):
print('close')
return obj
配置：
EXTENSIONS = {
'xiaohan.extends.MyExtension':500,
}
自定义命令
spider同级目录下创建文件夹commands写crawlall.py
在设置里面加 COMMANDS_MODULE = '项目名.commands'
from scrapy.commands import ScrapyCommand
from scrapy.utils.project import get_project_settings
class Command(ScrapyCommand):
requires_project = True
def syntax(self):
return '[options]'
def short_desc(self):
return 'Runs all of the spiders'
def run(self, args, opts):
spider_list = self.crawler_process.spiders.list()
for name in spider_list:
self.crawler_process.crawl(name, **opts.dict)
self.crawler_process.start()
浙公网安备 33010602011771号