用selenium 自动爬取某一本小说章节及其内容，并存入数据库中

from selenium import webdriver
import pymysql
from selenium.webdriver.support.ui import WebDriverWait     # 等待
from selenium.webdriver.support import expected_conditions as ec # 等待条件
from selenium.webdriver.common.by import By
import html
import _thread
from selenium.webdriver.chrome.options import Options

def ceil(x, y):
    if x % y == 0:              # 相除后为整数
        return int(x / y)
    else:                       # 相除有小数
        return int(x / y) + 1




# 创建一个浏览器
chrome_options = Options()
chrome_options.add_argument('--headless')
dr = webdriver.Chrome(chrome_options=chrome_options)

# 设置访问的网站
dr.get('https://doupocangqiong1.com/1/list_piaotian/')

# 获取所有的a标签
a = dr.find_elements_by_css_selector('.dirlist > li > a')

# 连接数据库
db = pymysql.connect("localhost", "root", "root", "selenium", charset='utf8')
# 获取游标
cursor = db.cursor()

for i in a:
    name = i.text
    href = i.get_attribute('href')
    sql = "INSERT INTO novel (name,href,content) VALUES ('%s','%s','%s')"%(name,href,'')
    cursor.execute(sql)         # 使用execute方法执行SQL语句
    db.commit()
dr.close()          # 关闭浏览器



def line(lineName, start, count):
    dr = webdriver.Chrome(chrome_options=chrome_options)         # 创建一个浏览器
    # 连接数据库
    db = pymysql.connect("localhost", "root", "root", "selenium", charset='utf8')
    # 获取游标
    cursor = db.cursor()

    sql = "SELECT id,href FROM novel LIMIT %s, %s"%(start, count)
    cursor.execute(sql)  # 使用execute方法执行SQL语句
    data = cursor.fetchall()  # 使用 fetchall() 方法获取所有数据
    for i in data:
        dr.get(i[1])
        # 放置等待
        WebDriverWait(dr, 5, 0.1).until_not(ec.text_to_be_present_in_element((By.CSS_SELECTOR, '#chaptercontent'),
                                                                            U'正在转码，请稍后......'))  # 等待dr浏览器10秒钟，每0.1秒钟问一次
        content = html.escape(dr.find_element_by_css_selector('#chaptercontent').text)
        # escape()将特殊字符转为特殊的编码格式，unescape()将编码格式转回特殊字符
        sql = "UPDATE novel SET content = '%s' WHERE id = %s" % (content, i[0])
        cursor.execute(sql)  # 使用execute方法执行SQL语句
        db.commit()
        print(lineName, '完成了', i[0], '的采集')
    dr.close()          # 关闭窗口
    dr.quit()           # 关闭浏览器
    cursor.close()
    db.close()
    print(lineName, '完成了采集')


def productLine(func, total, lineCount):
    every = ceil(total[0][0], lineCount)
    print('every', every)
    for i in range(lineCount):
        print('-------------', i)
        print(_thread.start_new_thread(func, ('line-' + str(i) + '', i * every, every)))


try:
    sql = 'SELECT COUNT(*) FROM novel'
    cursor.execute(sql)  # 使用execute方法执行SQL语句
    total = cursor.fetchall()  # 使用 fetchall() 方法获取所有数据
    print(total)

    productLine(line, total, 5)

except:
    print ("Error: unable to start thread")


while 1:
   pass