1.
采集目标地址 https://www.glass.cn/gongying/sellindex.aspx 网站比较简单,没什么大的需要注意的问题。
2.
通过分析测试 https://www.glass.cn/gongying/a_l_p1_ky/ 等价于目标采集网站首页,只需设置{}.format 翻页
这个完整比较简单,就是获取一下页码,再做一下翻页,循环采集页面跳转url,再进入url采集页面内容信息。
3.
采集数据过程及结果
#glass_gy.py # -*- coding: utf-8 -*- import scrapy import re from glass_gy_web.items import GlassGyWebItem class GlassGySpider(scrapy.Spider): name = 'glass_gy' allowed_domains = ['www.glass.cn'] start_urls = ['https://www.glass.cn/gongying/a_l_p1_ky/'] custom_settings = { 'DOWNLOAD_DELAY': 0.5, "ITEM_PIPELINES": { 'glass_gy_web.pipelines.MysqlPipeline': 300, }, "DOWNLOADER_MIDDLEWARES": { 'glass_gy_web.middlewares.GlassGyWebDownloaderMiddleware': 500, }, } def parse(self, response): link_urls = response.xpath("//div[@class='brandinfotxt z']/h4[@class='brandHTit clearfix']/a/@href").extract() for link_url in link_urls: # print(link_url) yield scrapy.Request(url=link_url, callback=self.parse_detail) # print('*'*100) # 翻页 pg_num = re.findall('共(.*?)页',response.text) # print(pg_num[0]) for i in range(2, int(pg_num[0])+1): url = 'https://www.glass.cn/gongying/a_l_p{}_ky/'.format(i) yield scrapy.Request(url=url, callback=self.parse) def parse_detail(self, response): item=GlassGyWebItem() # 产品名称 p_name = response.xpath("//div[@class='ProductDel']/h1/text()").extract_first() item['p_name'] = p_name # 数量 num = response.xpath("//table//tr/td[@class='bot'][1]/text()").extract_first() item['num'] = num # 价格 price = response.xpath("//table//tr/td[@class='bot'][2]/text()").extract_first() item['price'] = price # 供货总量 total_supply = re.findall('<li><span>供货总量:</span>(.*?)</li>',response.text) item['total_supply'] = total_supply[0] # 最小起订 _order = re.findall('<li><span>最小起订: </span>(.*?)</li>',response.text) item['_order'] = _order[0] # 发货地址 ship_add = re.findall('<li><span>发货地址: </span>(.*?)</li>',response.text) item['ship_add'] = ship_add[0] # 发货期限 dispatch = re.findall('<li><span>发货期限: </span>(.*?)</li>',response.text) item['dispatch'] = dispatch[0] # 物流运费 fare = re.findall('<li><span>物流运费: </span>(.*?)</li>',response.text) item['fare'] = fare[0] # 发布日期 release_date = re.findall('<li><span>发布日期:</span>(.*?)</li>',response.text) item['release_date'] = release_date[0] # 采购电话 purchase = re.findall('<div class="sell_tel clearfix"><span>采购电话:</span>(.*?)</div>',response.text) item['purchase'] = purchase[0] # 产品详情 content = response.xpath("//div[@id='pdetail']/text()").extract() item['content'] = content[-1].strip() # 公司名称 company = response.xpath("//div[@class='zcbw']/h2/a/text()").extract_first() item['company'] = company # 联系人 linkman = re.findall('<span>联系人:(.*?)</span>',response.text) item['linkman'] = linkman[0] # 手机 phone = re.findall('<li>手机:(.*?)</li>',response.text) item['phone'] = phone[0] # 电话 mobile = re.findall('<li>电话:(.*?)</li>',response.text) item['mobile'] = mobile[0] # 营业执照 try: licence = re.findall('<li>营业执照:(.*?) <img',response.text) licence = licence[0] except: licence = "" item['licence'] = licence # 经营模式 try: model = re.findall('<li>经营模式: (.*?)</li',response.text) model = model[0] except: model = "" item['model'] = model # 所在地区 dist = re.findall('<li>所在地区:(.*?)</li>',response.text) item['dist'] = dist[0] # 产品数量 quantity = re.findall('<li>产品数量:(.*?)</li>',response.text) item['quantity'] = quantity[0] # 玻璃金币 gold = re.findall('<li>玻璃金币:(.*?)</li>',response.text) item['gold'] = gold[0] yield item print('*'*100)
# items.py # -*- coding: utf-8 -*- # Define here the models for your scraped items # # See documentation in: # https://doc.scrapy.org/en/latest/topics/items.html import scrapy class GlassGyWebItem(scrapy.Item): # define the fields for your item here like: # name = scrapy.Field() # 产品名称 p_name = scrapy.Field() # 数量 num = scrapy.Field() # 价格 price = scrapy.Field() # 供货总量 total_supply = scrapy.Field() # 最小起订 _order = scrapy.Field() # 发货地址 ship_add = scrapy.Field() # 发货期限 dispatch = scrapy.Field() # 物流运费 fare = scrapy.Field() # 发布日期 release_date = scrapy.Field() # 采购电话 purchase = scrapy.Field() # 产品详情 content = scrapy.Field() # 公司名称 company = scrapy.Field() # 联系人 linkman = scrapy.Field() # 手机 phone = scrapy.Field() # 电话 mobile = scrapy.Field() # 营业执照 licence = scrapy.Field() # 经营模式 model = scrapy.Field() # 所在地区 dist = scrapy.Field() # 产品数量 quantity = scrapy.Field() # 玻璃金币 gold = scrapy.Field()
# piplines.py # -*- coding: utf-8 -*- # Define your item pipelines here # # Don't forget to add your pipeline to the ITEM_PIPELINES setting # See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html from scrapy.conf import settings import pymysql class GlassGyWebPipeline(object): def process_item(self, item, spider): return item # 数据保存mysql class MysqlPipeline(object): def open_spider(self, spider): self.host = settings.get('MYSQL_HOST') self.port = settings.get('MYSQL_PORT') self.user = settings.get('MYSQL_USER') self.password = settings.get('MYSQL_PASSWORD') self.db = settings.get(('MYSQL_DB')) self.table = settings.get('TABLE') self.client = pymysql.connect(host=self.host, user=self.user, password=self.password, port=self.port, db=self.db, charset='utf8') def process_item(self, item, spider): item_dict = dict(item) cursor = self.client.cursor() values = ','.join(['%s'] * len(item_dict)) keys = ','.join(item_dict.keys()) sql = 'INSERT INTO {table}({keys}) VALUES ({values})'.format(table=self.table, keys=keys, values=values) try: if cursor.execute(sql, tuple(item_dict.values())): # 第一个值为sql语句第二个为 值 为一个元组 print('数据入库成功!') self.client.commit() except Exception as e: print(e) print('数据已存在!') self.client.rollback() return item def close_spider(self, spider): self.client.close()
#settings.py # -*- coding: utf-8 -*- # Scrapy settings for glass_gy_web project # # For simplicity, this file contains only settings considered important or # commonly used. You can find more settings consulting the documentation: # # https://doc.scrapy.org/en/latest/topics/settings.html # https://doc.scrapy.org/en/latest/topics/downloader-middleware.html # https://doc.scrapy.org/en/latest/topics/spider-middleware.html BOT_NAME = 'glass_gy_web' SPIDER_MODULES = ['glass_gy_web.spiders'] NEWSPIDER_MODULE = 'glass_gy_web.spiders' # Crawl responsibly by identifying yourself (and your website) on the user-agent #USER_AGENT = 'glass_gy_web (+http://www.yourdomain.com)' # Obey robots.txt rules ROBOTSTXT_OBEY = False # mysql配置参数 MYSQL_HOST = "172.16.10.157" MYSQL_PORT = 3306 MYSQL_USER = "root" MYSQL_PASSWORD = "123456" MYSQL_DB = 'web_datas' TABLE = "glass_gy" # Configure maximum concurrent requests performed by Scrapy (default: 16) #CONCURRENT_REQUESTS = 32 # Configure a delay for requests for the same website (default: 0) # See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay # See also autothrottle settings and docs #DOWNLOAD_DELAY = 3 # The download delay setting will honor only one of: #CONCURRENT_REQUESTS_PER_DOMAIN = 16 #CONCURRENT_REQUESTS_PER_IP = 16 # Disable cookies (enabled by default) #COOKIES_ENABLED = False # Disable Telnet Console (enabled by default) #TELNETCONSOLE_ENABLED = False # Override the default request headers: #DEFAULT_REQUEST_HEADERS = { # 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', # 'Accept-Language': 'en', #} # Enable or disable spider middlewares # See https://doc.scrapy.org/en/latest/topics/spider-middleware.html #SPIDER_MIDDLEWARES = { # 'glass_gy_web.middlewares.GlassGyWebSpiderMiddleware': 543, #} # Enable or disable downloader middlewares # See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html DOWNLOADER_MIDDLEWARES = { 'glass_gy_web.middlewares.GlassGyWebDownloaderMiddleware': 543, } # Enable or disable extensions # See https://doc.scrapy.org/en/latest/topics/extensions.html #EXTENSIONS = { # 'scrapy.extensions.telnet.TelnetConsole': None, #} # Configure item pipelines # See https://doc.scrapy.org/en/latest/topics/item-pipeline.html ITEM_PIPELINES = { 'glass_gy_web.pipelines.GlassGyWebPipeline': 300, } # Enable and configure the AutoThrottle extension (disabled by default) # See https://doc.scrapy.org/en/latest/topics/autothrottle.html #AUTOTHROTTLE_ENABLED = True # The initial download delay #AUTOTHROTTLE_START_DELAY = 5 # The maximum download delay to be set in case of high latencies #AUTOTHROTTLE_MAX_DELAY = 60 # The average number of requests Scrapy should be sending in parallel to # each remote server #AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 # Enable showing throttling stats for every response received: #AUTOTHROTTLE_DEBUG = False # Enable and configure HTTP caching (disabled by default) # See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings #HTTPCACHE_ENABLED = True #HTTPCACHE_EXPIRATION_SECS = 0 #HTTPCACHE_DIR = 'httpcache' #HTTPCACHE_IGNORE_HTTP_CODES = [] #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
#middlewares.py # -*- coding: utf-8 -*- # Define here the models for your spider middleware # # See documentation in: # https://doc.scrapy.org/en/latest/topics/spider-middleware.html from scrapy import signals class GlassGyWebSpiderMiddleware(object): # Not all methods need to be defined. If a method is not defined, # scrapy acts as if the spider middleware does not modify the # passed objects. @classmethod def from_crawler(cls, crawler): # This method is used by Scrapy to create your spiders. s = cls() crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) return s def process_spider_input(self, response, spider): # Called for each response that goes through the spider # middleware and into the spider. # Should return None or raise an exception. return None def process_spider_output(self, response, result, spider): # Called with the results returned from the Spider, after # it has processed the response. # Must return an iterable of Request, dict or Item objects. for i in result: yield i def process_spider_exception(self, response, exception, spider): # Called when a spider or process_spider_input() method # (from other spider middleware) raises an exception. # Should return either None or an iterable of Response, dict # or Item objects. pass def process_start_requests(self, start_requests, spider): # Called with the start requests of the spider, and works # similarly to the process_spider_output() method, except # that it doesn’t have a response associated. # Must return only requests (not items). for r in start_requests: yield r def spider_opened(self, spider): spider.logger.info('Spider opened: %s' % spider.name) class GlassGyWebDownloaderMiddleware(object): # Not all methods need to be defined. If a method is not defined, # scrapy acts as if the downloader middleware does not modify the # passed objects. @classmethod def from_crawler(cls, crawler): # This method is used by Scrapy to create your spiders. s = cls() crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) return s def process_request(self, request, spider): # Called for each request that goes through the downloader # middleware. # Must either: # - return None: continue processing this request # - or return a Response object # - or return a Request object # - or raise IgnoreRequest: process_exception() methods of # installed downloader middleware will be called return None def process_response(self, request, response, spider): # Called with the response returned from the downloader. # Must either; # - return a Response object # - return a Request object # - or raise IgnoreRequest return response def process_exception(self, request, exception, spider): # Called when a download handler or a process_request() # (from other downloader middleware) raises an exception. # Must either: # - return None: continue processing this exception # - return a Response object: stops process_exception() chain # - return a Request object: stops process_exception() chain pass def spider_opened(self, spider): spider.logger.info('Spider opened: %s' % spider.name)