淘宝商品数据解析并保存

上一篇记录用selenium模拟登录淘宝，这一篇就接着上一篇记录一下之后如何解析数据并保存，其中数据提取用beautifulsoup，数据保存用mongodb。

具体步骤如下：

首先定位搜索框并输入要搜索的商品名称并点击搜索

input_query = self.wait.until(EC.presence_of_element_located((By.ID, 'q')))
search_button = self.wait.until(EC.presence_of_element_located((By.CLASS_NAME, 'search-button')))
input_query.send_keys(goods)
search_button.click()

然后用beautifulsoup提取如下图中的信息

soup = BeautifulSoup(browser.page_source, 'html.parser')
soup = soup.find_all(class_='J_MouserOnverReq')
for item in soup:
    goods_info = {}
    goods_info['price'] = item.select('.ctx-box > .row-1 > .price > strong')[0].get_text()
    goods_info['deal-cnt'] = item.select('.ctx-box > .row-1 > .deal-cnt')[0].get_text()
    goods_info['title'] = item.select('.ctx-box > .row-2 > a')[0].get_text().strip()
    goods_info['shop'] = item.select('.ctx-box > .row-3 > .shop > a')[0].get_text().strip()
    goods_info['location'] = item.select('.ctx-box > .row-3 > .location')[0].get_text()

接着将解析到的数据插入mongodb，结果如图所示

client = MongoClient(host='localhost', port=27017)
db = client['goods']
collection = db['macbook']
collection.insert_one(goods_info)

以下为登录并提取数据的全部代码：

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
import time
from bs4 import BeautifulSoup
from pymongo import MongoClient


driver_path = 'D:\\Program Files (x86)\\chromedriver.exe'


class TaoSearch:
    def __init__(self):
        self.url_login = 'https://login.taobao.com/member/login.jhtml'
        self.url_home = 'https://www.taobao.com'
        self.db = MongoClient(host='localhost', port=27017)['goods']
        self.cookies = {}
        options = webdriver.ChromeOptions()
        options.add_experimental_option('excludeSwitches', ['enable-automation'])
        self.browser = webdriver.Chrome(options=options, executable_path=driver_path)
        self.browser.maximize_window()
        self.wait = WebDriverWait(self.browser, 10)

    def login(self, username, password):
        self.browser.get(self.url_login)
        self.wait.until(EC.presence_of_element_located((By.ID, 'fm-login-id'))).send_keys(username)
        self.wait.until(EC.presence_of_element_located((By.ID, 'fm-login-password'))).send_keys(password)
        self.verify()
        time.sleep(3)
        home_page = self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '.site-nav-bd > ul.site-nav-bd-r > li.site-nav-home > div > a > span')))
        home_page.click()
        time.sleep(3)

    def search(self, goods):
        input_query = self.wait.until(EC.presence_of_element_located((By.ID, 'q')))
        input_query.send_keys(goods)
        search_button = self.wait.until(EC.presence_of_element_located((By.CLASS_NAME, 'search-button')))
        search_button.click()
        time.sleep(5)
        return goods, self.browser.page_source

    def parse_and_save(self, goods, source):
        collection = self.db[goods]
        soup = BeautifulSoup(source, 'html.parser')
        soup = soup.find_all(class_='J_MouserOnverReq')
        for item in soup:
            goods_info = {}
            goods_info['price'] = item.select('.ctx-box > .row-1 > .price > strong')[0].get_text()
            goods_info['deal-cnt'] = item.select('.ctx-box > .row-1 > .deal-cnt')[0].get_text()
            goods_info['title'] = item.select('.ctx-box > .row-2 > a')[0].get_text().strip()
            goods_info['shop'] = item.select('.ctx-box > .row-3 > .shop > a')[0].get_text().strip()
            goods_info['location'] = item.select('.ctx-box > .row-3 > .location')[0].get_text()
            collection.insert_one(goods_info)
            print(goods_info)

    def verify(self):
        button = self.browser.find_element_by_class_name('fm-btn')
        button.click()
        slider = self.wait.until(EC.presence_of_element_located((By.ID, 'nc_1_n1z')))
        ActionChains(self.browser).click_and_hold(on_element=slider).perform()
        ActionChains(self.browser).move_by_offset(xoffset=280, yoffset=0).perform()
        ActionChains(self.browser).release().perform()
        time.sleep(1)
        button.click()
        time.sleep(4)
        if self.browser.current_url == self.url_login:
            self.verify()

    def main(self，username, password, goods):
        self.login(username, password)
        goods_, source = self.search(goods)
        self.parse_and_save(goods_, source)


if __name__ == '__main__':
    t = TaoSearch()
    t.main('username', 'password', 'macbook')