爬取豌豆荚app数据

方法一：requests+bs4

 

import requests
from bs4 import BeautifulSoup
import re

# 1.发送请求
def get_page(url):
    response = requests.get(url)
    return response

# 2.解析数据
def parse_index(code):
    app_info = code.find_all(name='li', class_='card')
    return app_info

# 3.保存数据
def save_data(app):
    app_name = app.h2.a.text
    detail_url = app.h2.a.attrs['href']
    download_num = app.find(class_='install-count').text
    app_size = app.find(class_='meta').find_all(name='span')[2].attrs['title']
    data = f"""
    app名称：{app_name}
    详情页url：{detail_url}
    下载人数：{download_num}
    app大小：{app_size}
    \n
    """
    print(data)
    with open('豌豆荚.txt','a',encoding='utf-8') as f:
        f.write(data)

if __name__ == '__main__':
    url = 'https://www.wandoujia.com/category/6001'
    index_response = get_page(url)
    # print(url)
    index_code = BeautifulSoup(index_response.text,'lxml')
    app_list = parse_index(index_code)
    for app in app_list:
        save_data(app)
    print('写入成功......')

 

方法二：selenium

 

from selenium import webdriver
import time

driver = webdriver.Chrome()

try:
    driver.implicitly_wait(20)
    driver.get('https://www.wandoujia.com/category/6001')

    time.sleep(5)
    js_code = '''
                   window.scrollTo(0,5000)
                   '''
    driver.execute_script(js_code)

    time.sleep(5) # 信息加载，等待5s
    app_list = driver.find_elements_by_class_name('card')
    for app in app_list:
        # app名称
        app_name = app.find_element_by_css_selector('.app-title-h2 a').text

        # 详情页url
        detail_url = app.find_element_by_css_selector('.app-title-h2 a').get_attribute('href')

        # 下载人数和app大小
        download_num_size= app.find_element_by_class_name('meta').text
        #这两个信息都在同一个div里，app大小又没有class和id定位就一起输出，然后再用字符串提取数据方法分别提取

        app_content = f"""
        ========================================
            app名字：{app_name}
            详情页url：{detail_url}
            下载人数：{download_num_size[:8]}
            app大小：{download_num_size[11:]}
        ========================================
            \n
        """
        print(app_content)
        with open('豌豆荚app.txt','a',encoding='utf-8') as f:
            f.write(app_content)
    time.sleep(3)

finally:
    driver.close()