scrapy 爬去网页（1）

第一次爬去https://segmentfault.com/t/python?type=newest&page=1

首先定义爬去的字段：

  

class CnblogsItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    title = scrapy.Field()
    link = scrapy.Field()
    desc = scrapy.Field()
    listUrl = scrapy.Field()

　　编写爬虫：

#coding:utf-8
#! /usr/bin/python
'''
Author fiz
Date:2016-03-30 
'''

#coding:utf8
from scrapy.selector import HtmlXPathSelector
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor as  sle
from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.http import FormRequest, Request
from scrapy.selector import Selector
from Scrapy_demo.items import  *
from scrapy import log
from scrapy.utils.response import get_base_url

class SgfSpider(CrawlSpider):
    name = 'sgf'
    allowed_domains = ['segmentfault.com']
    start_urls = ['https://segmentfault.com/t/python?type=newest&page=1',]
    #此处注意?要转义
    rules = [ Rule(sle(allow=('/t/python\?type=newest&page=\d{1,}'),), follow=True,callback='parse_item1') ]



    def parse_item1(self, response):
        sel = Selector(response)

        items = []
        base_url = get_base_url(response)
        postTitle = sel.css('div.tab-content').css("section")#全部的问题数量每一页

        postCon = sel.css('div.postCon div.c_b_p_desc')
        # #标题、url和描述的结构是一个松散的结构，后期可以改进
        for index in range(len(postTitle)):
            item = CnblogsItem()
            #问题名称
            item['title'] = postTitle[index].css("a").xpath('text()').extract()[0]
            # item['link'] = 'https://segmentfault.com'+postTitle[index].css('a').xpath('@href').extract()[0]#提问人的主页链接
            #问题页面链接
            item['link'] = 'https://segmentfault.com'+postTitle[index].css("h2.title").css('a').xpath('@href').extract()[0]
            #当前爬去的页面
            item['listUrl'] = base_url

            item['desc'] = postTitle[index].css("div.answers ").xpath("text()").extract()[0]
            #print base_url + "********\n"
            items.append(item)
        return items

　　编写PIPlines

#coding:utf-8
#! /usr/bin/python
'''
Author fiz
Date:2016-03-31
'''

import pymongo
from scrapy.conf import settings
from scrapy.exceptions import DropItem
from scrapy import log
class MongoDBPipeline( object ):
   def __init__( self ):

     connection = pymongo.MongoClient()
     db = connection[settings[ 'MONGODB_DB' ]]
     self .collection = db[settings[ 'MONGODB_COLLECTION' ]]

   def process_item( self , item, spider):
     valid = True
     for data in item:
       if not data:
         valid = False
         raise DropItem( "Missing {0}!" . format (data))
     if valid:
       self .collection.insert( dict (item))
       log.msg( "Question added to MongoDB database!" ,
           level = log.DEBUG, spider = spider)
     return item

　　结果爬去了3456条数据源码在https://github.com/FizLBQ/SpiderPython/tree/Scrapy_demo