基础

1 安装scrapy

2 pip install wheel

3 下载Twisted https://www.lfd.uci.edu/~gohlke/pythonlibs/

4 pip install Twisted-xxxxxx

5 pip install scrapy

创建工程

scrapy startproject xxxpro

scrapy genspider spidername www.xxx.com

运行

scrapy crawl qiubai

extract()

 1 def parse(self, response):
 2     title_list=response.xpath("/html/body/main/div/div/div//posts")
 3     for title in title_list:
 4         title_name=title.xpath(".//div[2]/h2/a/text()")[0].extract()
 5         title_eyes=title.xpath(".//item[@class='meta-view']/text()")[0].extract()
 6        
 7         # print(title_name,title_eyes)
 8         
 9         item=TestproItem()#前面需要引入文件item
10         item['title_name']=title_name
11         item['title_eyes'] = title_eyes
12         yield item

pipelines.py 管道文件一个管道类存一种方式

TXT管道

 1 class TestproPipeline:
 2     fp=None
 3     def open_spider(self,spider):
 4         print("开始爬虫")
 5         self.fp =open('./qiu.txt','w',encoding='utf-8')
 6 
 7     def process_item(self, item, spider):
 8         title_name = item['title_name']
 9         title_eyes = item['title_eyes']
10         # print(title_name,title_eyes)
11         self.fp.write(title_name+":"+title_eyes+"\n")
12         return item  #两种管道必须都加返回，否则第二个管道无法获取item
13     def close_spider(self,spider):
14         print("爬虫结束")
15         self.fp.close()

mysql 管道

 1 class mysqlPipeline:
 2     conn=None
 3     cursor=None
 4     def open_spider(self, spider):
 5         print("开始爬虫")
 6         self.conn=pymysql.Connect(host="127.0.0.1",user="root",password="root",database="xijing",port=3306)
 7 
 8     def process_item(self, item, spider):
 9         title_name = item['title_name']
10         title_eyes = item['title_eyes']
11         # print(title_name,title_eyes)
12         self.cursor=self.conn.cursor()
13         try:
14             self.cursor.execute('insert into water values ("%s","%s")'%(title_name,title_eyes))
15             self.conn.commit()
16         except Exception as e:
17             print(e)
18             self.conn.rollback()
19         return item   #两种管道必须都加返回，否则第二个管道无法获取item
20 
21     def close_spider(self, spider):
22         self.cursor.close()
23         self.conn.close()
24         print("爬虫结束")

posted @ 2022-06-12 11:19 fbhell 阅读(12) 评论(0) 收藏举报

刷新页面返回顶部