1.安装环境
http://pan.baidu.com/s/1bnAKBSz
修改环境变量path.添加 c:\python27;c:\python27\Script;
2.如需使用mysql等数据库请自行安装
3.安装好环境后，进入命令行，进入到工作目录，使用以下命令创建工程(工程名以doubanmoive为例)
scrapy startproject doubanmoive
4.做一系列修改使用以下命令运行项目
scrapy crawl doubanmoive
5.scrapy的一些注意地方
(1)步骤3完成以后，项目的目录应该是这样的（根目录各人不同）
D:\WEB\Python\doubanmoive>tree /f
Folder PATH listing for volume Data
Volume serial number is 00000200 34EC:9CB9
D:.
│ scrapy.cfg
│
└─doubanmoive
│ items.py
│ pipelines.py
│ settings.py
│ __init__.py
│
└─spiders
__init__.py
moive_spider.py
moive_spider.pyc
(2)这些文件主要功能为：
- doubanmoive/items.py：定义需要获取的内容字段，类似于实体类。
- doubanmoive/pipelines.py：项目管道文件，用来处理Spider抓取的数据。
- doubanmoive/settings.py：项目配置文件
- doubanmoive/spiders/moive_spider.py：放置spider的目录
(3)demo
doubanmoive/items.py
from scrapy.item import Item, Field class DoubanmoiveItem(Item): name=Field()#电影名 year=Field()#上映年份 score=Field()#豆瓣分数 director=Field()#导演 classification=Field()#分类 actor=Field()#演员 img=Field()#剧照
doubanmoive/pipelines.py
# -*- coding: utf-8 -*-
from scrapy import log
from twisted.enterprise import adbapi
from scrapy.http import Request
from scrapy.selector import HtmlXPathSelector
import urllib
import MySQLdb
import MySQLdb.cursors
class DoubanmoivePipeline(object):
def __init__(self):
#定义mysql数据，db:mysql database name,user: mysql username,passwd:mysql password
self.dbpool = adbapi.ConnectionPool('MySQLdb',
db = 'python',
user = 'root',
passwd = 'root',
cursorclass = MySQLdb.cursors.DictCursor,
charset = 'utf8',
use_unicode = False
)
def process_item(self, item, spider):
query = self.dbpool.runInteraction(self._conditional_insert, item)
query.addErrback(self.handle_error)
return item
def _conditional_insert(self,tx,item):
#检查数据库中是否存在该名字的电影
tx.execute("select * from doubanmoive where m_name= %s",(item['name'][0],))
result=tx.fetchone()
# log.msg(result,level=log.DEBUG)
print result
if result:
log.msg("Item already stored in db:%s" % item,level=log.DEBUG)
else:
#处理过多的主演和不符合的时间格式
classification=actor=''
lenClassification=len(item['classification'])
lenActor=len(item['actor'])
for n in xrange(lenClassification):
classification+=item['classification'][n]
if n<lenClassification-1:
classification+='/'
for n in xrange(lenActor):
actor+=item['actor'][n]
if n<lenActor-1:
actor+='/'
#获取海报下载地址
site= item['img'][0]
#截取海报地址的最后一个/,生成本地的文件名
str = site.split('/');
print str
path = str[-1]
print 'local img path %s'%(path)
#开始下载海报
print '--------------------download img %s'%(site)
data = urllib.urlopen(site).read()
newfile = open(path,"wb")
newfile.write(data)
newfile.close()
#将这些数据插入到数据库里面
tx.execute(\
"insert into doubanmoive (m_name,m_year,m_score,m_director,m_classification,m_actor,m_img,m_local_img) values (%s,%s,%s,%s,%s,%s,%s,%s)",\
(item['name'][0],item['year'][0],item['score'][0],item['director'][0],classification,actor,site,path))
# log.msg("Item stored in db: %s" % item, level=log.DEBUG)
def handle_error(self, e):
log.err(e)
doubanmoive/spiders
# -*- coding: utf-8 -*-
from scrapy.selector import HtmlXPathSelector
from scrapy.contrib.spiders import CrawlSpider,Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from doubanmoive.items import DoubanmoiveItem
class MoiveSpider(CrawlSpider):
name="doubanmoive"
#允许访问的domain
allowed_domains=["movie.douban.com"]
#开始的地址
start_urls=["http://movie.douban.com/top250"]
#定义规则：允许访问的地址正则表达式，申明爬虫回调方法
rules=[
Rule(SgmlLinkExtractor(allow=(r'http://movie.douban.com/top250\?start=\d+.*'))),
Rule(SgmlLinkExtractor(allow=(r'http://movie.douban.com/subject/\d+')),callback="parse_item"),
]
#rules申明的爬虫回调方法
def parse_item(self,response):
sel=HtmlXPathSelector(response)
item=DoubanmoiveItem()
#定义获取item字段的xpath表达式，完成item解析赋值
item['name']=sel.xpath('//*[@id="content"]/h1/span[1]/text()').extract()
item['year']=sel.xpath('//*[@id="content"]/h1/span[2]/text()').re(r'\((\d+)\)')
item['score']=sel.xpath('//*[@id="interest_sectl"]/div/p[1]/strong/text()').extract()
item['director']=sel.xpath('//*[@id="info"]/span[1]/a/text()').extract()
item['classification']= sel.xpath('//span[@property="v:genre"]/text()').extract()
item['actor']= sel.xpath('//*[@id="info"]/span[3]/a[1]/text()').extract()
# item['img']= sel.xpath('//*[@id="content"]/div/div[1]/ol/li[1]/div/div[1]/a/img/text()').extract()
# item['img']= sel.select('//ol/li/div/div/a/img/@src').extract()
item['img']= sel.select('//a/img/@src').extract()
return item
doubanmoive/settings.py
# -*- coding: utf-8 -*-
# Scrapy settings for doubanmoive project
#
# For simplicity, this file contains only the most important settings by
# default. All the other settings are documented here:
#
# http://doc.scrapy.org/en/latest/topics/settings.html
#
BOT_NAME = 'doubanmoive'
SPIDER_MODULES = ['doubanmoive.spiders']
NEWSPIDER_MODULE = 'doubanmoive.spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'doubanmoive (+http://www.yourdomain.com)'
ITEM_PIPELINES={
'doubanmoive.pipelines.DoubanmoivePipeline':400,
}
LOG_LEVEL='DEBUG'
DOWNLOAD_DELAY = 2
RANDOMIZE_DOWNLOAD_DELAY = True
USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.54 Safari/536.5'
COOKIES_ENABLED = True
运行完成以后，数据库结果
