scrapy 例：新浪网分类资讯爬虫

items.py

import scrapy

class SinaspiderItem(scrapy.Item):
    # define the fields for your item here like:
    #大类的标题和url
    parentUrls = scrapy.Field()
    parentTitle=scrapy.Field()

    #小类的标题和url
    subUrls=scrapy.Field()
    subTitle=scrapy.Field()

    #小类目录存储路径
    subFilename=scrapy.Field()

    #小类下的子链接
    sonUrls=scrapy.Field()

    #文章标题和内容
    title=scrapy.Field()
    content=scrapy.Field()

 

spiders/sina.py

# -*- coding: utf-8 -*-
import scrapy
from ..items import SinaspiderItem
import os

class SinaSpider(scrapy.Spider):
    name = 'sina'
    # allowed_domains = ['news.sina.com.cn']   #不能这样写啊姐，如果只想爬取新闻大类，可以这样写就不会爬取别的大类了
    allowed_domains = ['sina.com.cn']
    start_urls = ['http://news.sina.com.cn/guide/']

    def parse(self, response):
        items=[]

        #所有大类的urls和title
        parentUrls=response.xpath("//h3[@class='tit02']/a/@href").extract()
        parentTitle=response.xpath("//h3[@class='tit02']/a/text()").extract()

        #所有小类的urls和title
        subUrls = response.xpath("//ul[@class='list01']/li/a/@href").extract()
        subTitle = response.xpath("//ul[@class='list01']/li/a/text()").extract()

        #爬取所有大类
        for i in range(0,len(parentTitle)):
            #指定大类目录的路径和目录名
            parentFilename='./data/'+parentTitle[i]

            # 如果目录不存在，则创建目录
            if not os.path.exists(parentFilename):
                os.makedirs(parentFilename)

            #爬取所有小类
            for j in range(0,len(subUrls)):
                item=SinaspiderItem()

                # 保存大类的title和urls
                item['parentUrls']=parentUrls[i]
                item['parentTitle']=parentTitle[i]

                # 检查小类的url是否以同类别大类url开头，如果是返回True
                if_belong=subUrls[j].startswith(parentUrls[i])

                # 如果属于本大类，将存储目录放在本大类目录下
                if if_belong:
                    subFilename=parentFilename+'/'+subTitle[j]

                    # 如果目录不存在，则创建目录
                    if not os.path.exists(subFilename):
                        os.makedirs(subFilename)

                    # 存储 小类url、title和filename字段数据
                    item['subUrls']=subUrls[j]
                    item['subTitle']=subTitle[j]
                    item['subFilename']=subFilename

                    items.append(item)

        # 发送每个小类url的Request请求，得到Response连同包含meta数据 一同交给回调函数 second_parse 方法处理
        for item in items:
            # print(item['parentUrls']) #将所有的parentUrls都拿到了
            yield scrapy.Request(url=item['subUrls'],meta={'meta_1':item},callback=self.second_parse)

    def second_parse(self,response):
        # 提取每次Response的meta数据
        meta_1=response.meta['meta_1']

        # 取出小类里所有子链接
        sonUrls=response.xpath("//a/@href").extract()
        # print(sonUrls)

        items=[]
        for i in range(0,len(sonUrls)):
            # 检查每个链接是否以大类url开头、以.shtml结尾，如果是返回True
            if_belong=sonUrls[i].startswith(meta_1['parentUrls']) and sonUrls[i].endswith('.shtml')

            # 如果属于本大类，获取字段值放在同一个item下便于传输
            if if_belong:
                item=SinaspiderItem()
                item['parentUrls'] = meta_1['parentUrls']
                item['parentTitle'] = meta_1['parentTitle']
                item['subUrls'] = meta_1['subUrls']
                item['subTitle'] = meta_1['subTitle']
                item['subFilename'] = meta_1['subFilename']
                item['sonUrls'] = sonUrls[i]

                items.append(item)
        # print(items)
        for item in items:
            # 发送每个小类下子链接url的Request请求，得到Response后连同包含meta数据 一同交给回调函数 detail_parse
            # print(item['sonUrls'])
            yield scrapy.Request(url=item['sonUrls'],meta={'meta_2':item},callback=self.detail_parse)

    # 数据解析方法，获取文章标题和内容
    def detail_parse(self,response):
        item=response.meta['meta_2']
        # print(item)
        title=response.xpath("//h1[@class='main-title']/text() | //div[@class='blkContainerSblk']/h1/text()").extract()  #同时满足
        conten_list=response.xpath("//div[@class='article']/p/text()").extract()

        # 将p标签里的文本内容合并到一起
        contents=""
        for content in conten_list:
            contents+=content

        item['title']=title
        item['content']=contents.strip()
        # print(item)

        yield item

 

pipelines.py

# 保存日志信息的文件名
LOG_FILE='sina.log'
# 保存日志等级，低于|等于此等级的信息都被保存
LOG_LEVEL='DEBUG'

DEFAULT_REQUEST_HEADERS = {
  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36",
  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  'Accept-Language': 'en',
}

ITEM_PIPELINES = {
   'sinaSpider.pipelines.SinaspiderPipeline': 300,
}

 

settings.py

class SinaspiderPipeline(object):
    def process_item(self, item, spider):
        sonUrls=item['sonUrls']
        filename=sonUrls[7:-6].replace('/',"_")+'.txt'
        with open(item['subFilename']+'/'+filename,'w') as f:
            f.write(item['content'])
        return item