爬虫实战: 爬取当当网top500书籍

1.这个好像是爬虫入门必备项目，练练手

　　练习网站：http://bang.dangdang.com/books/fivestars 

 

2.requests + bs4模式，因为这个网站比较简单，不多说废话了。

   （此次下载的内容没有输出到文本文件中，而是存储到mongodb数据库中）

#!/usr/bin/env python
#-*- coding:utf-8 -*-


'''爬取当当网top500书籍
'''

import requests
from bs4 import BeautifulSoup
from pymongo import MongoClient


headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36'
}

class SpiderDangdang(object):
    
    def __init__(self,url):
        self.url = url
        
        
    def get_collection(self):
        '''将下载的数据存入mongodb数据库
        '''
        client = MongoClient('localhost', 27017)
        database = client.spider
        collection = database.dangdang
        return collection
    
            
    def get_response(self):
        try:
            response = requests.get(self.url,headers=headers)
            response.raise_for_status
            response.encoding = response.apparent_encoding
            return response.text 
        except Exception as e:
            print('aa',e)
            return 'None'
    
    
    def get_soup(self,response):
        try:
            soup = BeautifulSoup(response,'html.parser')
        except:
            soup = BeautifulSoup(response,'html.parser')
        return soup
    
    
    def get_items(self,soup):
        items = soup.select('div.bang_list_box>ul>li')
        return items
    
    
    def get_item_content(self,item):
        num = item.select('div.list_num')[0].text.strip()
        name = item.select('div.name')[0].text.strip()
        star = item.select('div.star')[0].text.strip()
        author = item.select('div.publisher_info')[0].text.strip()
        try:
            if item.select('div.price>p>span.price_n'):
                price_n = item.select('div.price>p>span.price_n')[0].text.strip()
            else:
                price_n = 'None'
        except:
            price_n = 'None'
            
        try:
            if item.select('div.price>p>span.price_r'):
                price_r = item.select('div.price>p>span.price_r')[0].text.strip()
            else:
                price_r = 'None'
        except:
            price_r = 'None'
            
        try:
            if item.select('div.price>p>span.price_s'):
                price_s = item.select('div.price>p>span.price_s')[0].text.strip()
            else:
                price_s = 'None'
        except:
            price_s = 'None'

        content = {
            '排名': num,
            '书名': name,
            '评分': star,
            '作者': author,
            '当前价格': price_n,
            '原价': price_r,
            '折扣': price_s
        }
        return content 
    
    
    def start(self):
        collection = self.get_collection()
        response = self.get_response()
        soup = self.get_soup(response)
        items = self.get_items(soup)
        for item in items:
            content = self.get_item_content(item)
            query = {'排名': content['排名']}
            if collection.find_one(query):
                print('\033[1;31m该item已经存在，不进行存储\033[0m')
            else:
                collection.insert_one(content)
                print('\033[1;32m该item是新的, 进行存储\033[0m')
            
    

if __name__ == '__main__':
    
    urls = ['http://bang.dangdang.com/books/fivestars/1-{page}'.format(**locals()) for page in range(1,26)]
    
    for page,url in enumerate(urls):
        print('\033[1;33m开始爬取第{page}页\033[0m'.format(page=page+1))
        ss = SpiderDangdang(url)
        ss.start()

 

3.注意数据存储时，进行相应的判断，去掉已经存入过的item；

简单的for循环语句，建议使用列表推导式/字典/元组推导式等，简单明了

最后，输出必要的提示字符，了解程序目前运行的阶段