数据存储 mysql
数据存储 mysql
一、MySQL 同步存储
二、MySQL 异步存储
1 from scrapy import log 2 import pymysql 3 import pymysql.cursors 4 import codecs 5 from twisted.enterprise import adbapi 6 7 class MysqlTwistedPipeline(object): 8 9 @classmethod 10 def from_settings(cls, settings): 11 dbargs = dict( 12 host=settings['MYSQL_HOST'], 13 db=settings['MYSQL_DBNAME'], 14 user=settings['MYSQL_USER'], 15 passwd=settings['MYSQL_PASSWD'], 16 port=settings['MYSQL_PORT'], 17 charset='utf8', 18 cursorclass=pymysql.cursors.DictCursor, 19 use_unicode=True, 20 ) 21 dbpool = adbapi.ConnectionPool('pymysql', **dbargs) 22 return cls(dbpool) 23 24 25 def __init__(self,dbpool): 26 self.dbpool=dbpool 27 28 #pipeline默认调用 29 def process_item(self, item, spider): 30 d=self.dbpool.runInteraction(self._conditional_insert, item, spider)#调用插入的方法 31 log.msg("-------------------连接好了-------------------") 32 d.addErrback(self._handle_error,item,spider)#调用异常处理方法 33 d.addBoth(lambda _: item) 34 return d 35 36 def _conditional_insert(self, conn, item, spider): 37 log.msg("-------------------打印-------------------") 38 39 conn.execute("insert into MovieInfo (poster, name, director, actor, tag_char, score, critics_num,) values(%s, %s, %s,%s,%s,%s,%s,)", 40 (item['cover'], item['title'], item['region'], item['years'], item['types'], item['score'], item['comment_num'],)) 41 log.msg("-------------------一轮循环完毕-------------------") 42 def _handle_error(self, failue, item, spider): 43 print(failue)