scrapy 例：爬取豆瓣电影-数据存放mongo+中间件

一. 新建项目(scrapy startproject)

scrapy startproject doubanSpider

 

二、明确目标(doubanSpider/items.py)

import scrapy

class DoubanspiderItem(scrapy.Item):
    # define the fields for your item here like:
    name = scrapy.Field()
    content=scrapy.Field()
    num=scrapy.Field()
    quote=scrapy.Field()

 

三、制作爬虫 （spiders/douban.py）

　1、scrapy genspider douban "movie.douban.com"

   2、打开doubanSpider/spider目录里的 douban.py，代码如下：

# -*- coding: utf-8 -*-
import scrapy
from ..items import DoubanspiderItem

class DoubanSpider(scrapy.Spider):
    name = 'douban'
    allowed_domains = ['movie.douban.com']

    url="https://movie.douban.com/top250?start="
    offset=0
    start_urls = [url+str(offset)]

    def parse(self, response):
        node_list=response.xpath("//div[@class='info']")
        for node in node_list:
            item = DoubanspiderItem()
            item['name'] = node.xpath('./div[1]/a/span[1]/text()').extract()[0]
            content = []
            content.append(node.xpath('./div[2]/p[1]/text()').extract()[0].strip())
            content.append(node.xpath('./div[2]/p[1]/text()[last()]').extract()[0].strip())  # 使用text()[last()]方法，取文本<br>后内容
            item['content'] = content
            item['num'] = node.xpath('./div[2]/div/span[2]/text()').extract()[0]
            item['quote'] = node.xpath('./div[2]/p[2]/span/text()').extract()[0]

            yield item

        if self.offset<100:
            self.offset+=25

        yield scrapy.Request(self.url+str(self.offset),callback=self.parse)

 

四、存储内容 （pipelines.py)

修改settings.py以下几个地方(可将日志打印在文件里)：

LOG_FILE = "douban.log"
LOG_LEVEL = "DEBUG"

ROBOTSTXT_OBEY = True

#除非特殊需要，禁用cookies，防止某些网站根据Cookie来封锁爬虫。
COOKIES_ENABLED = False

# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#添加自己编写的下载中间件类
DOWNLOADER_MIDDLEWARES = {
   'doubanSpider.middlewares.RandomUserAgent': 100,
   'doubanSpider.middlewares.RandomProxy': 200,}

USER_AGENTS = [
    'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0)',
    'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.2)',
    'Opera/9.27 (Windows NT 5.2; U; zh-cn)',
    'Opera/8.0 (Macintosh; PPC Mac OS X; U; en)',
    'Mozilla/5.0 (Macintosh; PPC Mac OS X; U; en) Opera 8.0',
    'Mozilla/5.0 (Linux; U; Android 4.0.3; zh-cn; M032 Build/IML74K) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30',
    'Mozilla/5.0 (Windows; U; Windows NT 5.2) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13'
]

PROXIES = [
        {"ip_port" :"124.237.83.14:53281", "user_passwd" : ""},
        #{"ip_prot" :"121.42.140.113:16816", "user_passwd" : "888888"}
        #{"ip_prot" :"121.42.140.113:16816", "user_passwd" : ""}
        #{"ip_prot" :"121.42.140.113:16816", "user_passwd" : ""}
]

# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
   'doubanSpider.pipelines.DoubanspiderPipeline': 300,
}

# MONGODB 主机名
MONGO_HOST="127.0.0.1"
# MONGODB 端口号
MONGO_PORT=27017
# 数据库名称
MONGO_DBNAME="Douban"
# 存放数据的表名称
MONGO_SHEETNAME="doubanmovies"

编写pipelines.py文件(数据存入mongodb)

from scrapy.utils.project import get_project_settings  #导入settings.py文件
import pymongo

class DoubanspiderPipeline(object):
    def __init__(self):
        host = get_project_settings().get('MONGO_HOST')
        port = get_project_settings().get('MONGO_PORT')
        dbname = get_project_settings().get('MONGO_DBNAME')
        sheetname = get_project_settings().get('MONGO_SHEETNAME')

        # 创建MONGODB数据库链接
        client = pymongo.MongoClient(host=host, port=port)
        # 指定数据库
        db = client[dbname]
        # 存放数据的数据库表名
        self.sheet = db[sheetname]

    def process_item(self, item, spider):
        self.sheet.insert(dict(item))
        return item

mongodb命令如下:

 

 

 

 

 编写middlewares.py文件

import random
import base64
from scrapy.utils.project import get_project_settings  #导入settings.py文件

# 随机的User-Agent
class RandomUserAgent():
    def process_request(self,request,spider):
        useragent=random.choice(get_project_settings().get('USER_AGENTS'))
        # print(useragent)
        request.headers.setdefault("User-Agent",useragent)

class RandomProxy():
    def process_request(self, request, spider):
        proxy=random.choice(get_project_settings().get('PROXIES'))

        if proxy['user_passwd'] is None:
            # 没有代理账户验证的代理使用方式
            request.meta['proxy']='http://'+proxy['ip_port']
        # else:
        #     # 对账户密码进行base64编码转换
        #     base64_userpasswd=base64.b64encode(proxy['user_passwd'])
        #     # 对应到代理服务器的信令格式里
        #     request.headers['Proxy-Authorization'] = 'Basic ' + base64_userpasswd
        #
        #     request.meta['proxy'] = "http://" + proxy['ip_port']

 

命令执行：