使用scrapy框架爬取豆瓣电影TOP250

创建一个项目

scrapy startproject dbdianying

创建爬虫
scrapy genspider douban douban.com

设置settings.py

1.USER_AGENT

2.

ROBOTSTXT_OBEY = False

DOWNLOAD_DELAY = 3

ITEM_PIPELINES = {
   'dbdianying.pipelines.ImagePipeline': 300,
}
#定义图片保存的位置
IMAGES_STORE = "d:/doubandianying"

 

douban.py#爬虫文件

import scrapy


class DoubanSpider(scrapy.Spider):
    name = 'douban'
    allowed_domains = ['douban.com']
    start_urls = ['https://movie.douban.com/top250']

    def parse(self, response):
        movie_urls = response.xpath('//div[@class="pic"]/a/@href')
        yield from scrapy.follow_all(movie_urls, callback=self.parse_movie)

        next_url = response.xpath('//span[@class="next"]/a/@href').get()
        yield scrapy.Request(response.urljoin(next_url), callback=self.parse)

    def parse_movie(self, response):
        name = response.xpath('//div[@id="content"]/h1/span[1]/text()').extract_first()
        director = response.xpath('//span[@class="attrs"]/a[@rel="v:directedBy"]/text()').extract_first()
        type = ''.join(response.xpath('//span[@property="v:genre"]/text()').extract())
        score = response.xpath('//div[@typeof="v:Rating"]/strong/text()').extract_first()
        img_url = response.xpath('//div[@id="mainpic"]/a/img/@src').extract_first()
        yield {
            'name': name,
            'director': director,
            'type': type,
            'score': score,
            'img_url': img_url,
        }

pipeline.py

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
from scrapy.pipelines.images import ImagesPipeline
from scrapy import Request

# class DbdianyingPipeline:
#     def open_spider(self,spider):
#         self.filename = open('movie.txt','a',encoding='utf-8')
#     def process_item(self, item, spider):
#         info = item['name'] + '\n' + item['director'] + '\n' + item['type'] + '\n' + item['score'] + '\n' + item['img_url'] + '\n'
#         self.filename.write(info)
#         self.filename.flush()
#         print(item)
#         return item
#     def close_spider(self,spider):
#         self.filename.close()
class ImagePipeline(ImagesPipeline):
    def get_media_requests(self, item, info):
        yield Request(item['img_url'],meta={'name': item['name']})

    def file_path(self, request, response=None, info=None):
        name = request.meta['name']
        name = name.replace(':', '：')
        return name + '.jpg'

数据导出执行

 scrapy crawl zongheng -o movie.csv