scrapy框架爬取17173游戏

1.任务：

scrapy框架爬取17173游戏排行榜，内容包括游戏名称、票数，并保存

2.爬虫代码 -- a17173.py

 

import scrapy
import json

class A17173Spider(scrapy.Spider):
    name = '17173'
    allowed_domains = ['17173.com']
    start_urls = ['http://top.17173.com/default-index.html?oper_status=1&game_frame=0&game_type=0&game_theme=0&game_feature=0&page={}'.format(num) for num in range(1,6)]
    def parse(self, response):
        newgame_ranks = response.xpath('//div[@class="main-c1"]//div[@class="c1"]/em/text()').extract()
        newgame_names = response.xpath('//div[@class="main-c1"]//div[@class="con"]/a/text()').extract()
        newgame_votess = [x.strip() for x in response.xpath('//div[@class="main-c1"]//div[@class="item-in"]/div[@class="c3"]/text()').extract()]
        hotgame_ranks = response.xpath('//div[@class="main-c2"]//div[@class="c1"]/em/text()').extract()
        hotgame_names = response.xpath('//div[@class="main-c2"]//div[@class="con"]/a/text()').extract()
        hotgame_votess = [x.strip() for x in response.xpath('//div[@class="main-c2"]//div[@class="item-in"]/div[@class="c3"]/text()').extract()]

        for newgame_rank, newgame_name, newgame_votes, hotgame_rank, hotgame_name, hotgame_votes in zip(newgame_ranks, newgame_names, newgame_votess, hotgame_ranks, hotgame_names, hotgame_votess):
            yield {
                'newgame_rank' : newgame_rank,
                'newgame_name': newgame_name,
                'newgame_votes': newgame_votes,
                'hotgame_rank': hotgame_rank,
                'hotgame_name': hotgame_name,
                'hotgame_votes': hotgame_votes
            }
        print(type(hotgame_votess))

 

3.pipelines.py

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
from itemadapter import ItemAdapter


class GamePipeline:
    def open_spider(self,spider):
        self.filename1 = open('新游期待榜.txt','w',encoding='utf-8')
        self.filename2 = open('热门游戏榜.txt','w',encoding='utf-8')


    def process_item(self, item, spider):
        info1 = item['newgame_rank'] + '\t' + item['newgame_name'] + '\t' + item['newgame_votes'] + '\n'
        self.filename1.write(info1)
        self.filename1.flush()

        info2 = item['hotgame_rank'] + '\t' + item['hotgame_name'] + '\t' + item['hotgame_votes'] + '\n'
        self.filename2.write(info2)
        self.filename2.flush()

        return item


    def close_spider(self,spider):
        self.filename1.close()
        self.filename2.close()

4.结果