selenium-浙江新闻网-简单示例

# coding: utf-8
import html
import json
import os
import re
import sys
from datetime import datetime
from pprint import pprint

import parsel
import requests
import schedule
from selenium.webdriver import Chrome
from selenium.webdriver.chrome.options import Options
import random
import time
import threading
from concurrent.futures import ThreadPoolExecutor


class WebDriver():
    def __init__(self, url):
        self.url = url
        self.option = Options()
        self.option.add_argument('--disable-blink-features=AutomationControlled')
        self.option.add_argument('headless')
        self.option.add_argument('--disable-gpu')
        self.web = Chrome(options=self.option)

    def run(self):
        self.web.get(url=self.url)
        self.web.maximize_window()
        self.web.implicitly_wait(20)
        time.sleep(1)

        for i in range(0, 10000, random.randint(100,150)):
            js = f"document.querySelector('#zjxw_pc').scrollTo(0, {i})"
            self.web.execute_script(js)
            time.sleep(random.random())
        return self.pageSource()

    def pageSource(self):
        return self.web.page_source


class Spider(WebDriver):
    def parseHomePage(self, htmlSource):
        selector = parsel.Selector(htmlSource)
        divs = selector.css('div.channelList_article_container__3dui5 div.listItems_article_item__3B3fT')
        articleUrls = []
        for div in divs:
            articleUrl = div.css('a::attr(href)').get()
            articleUrls.append(articleUrl)
        return articleUrls

    def paseArticlePage(self, articleUrl):
        from faker import Faker
        headers = {'user-agent': Faker().user_agent()}
        resp = requests.get(url=articleUrl, headers=headers)
        time.sleep(random.random())

        selectoe = parsel.Selector(resp.text)
        try:
            json_data = json.loads(selectoe.css('script#__NEXT_DATA__::text').get())
            articleData = json_data['props']['pageProps']['article']
        except TypeError:
            return None

        docTitle = articleData['docTitle']
        publishedAt = articleData['publishedAt']
        dateObj = datetime.fromtimestamp(int(str(publishedAt)[0:10]))
        publishedAt = dateObj.strftime('%Y-%m-%d %H:%M:%S')
        content = articleData['content']
        source = articleData['source']
        author = articleData['author']
        if author:
            author = f'<p>作者：{author}</p>'
        else:
            author = ''
        webLink = articleData['webLink']
        if webLink != '':
            video = ''.join(selectoe.css('video').getall())
            video = html.unescape(video)
            video = f'<div div align="center" style="text-align: left;">{video}</div>'
            content = video + content

        date = publishedAt.split(' ', 1)[0]
        pub_date = publishedAt + '@' + source
        column_url = self.url
        head = f'<p class="column_url"><a href="{column_url}">栏目地址：{column_url}</a></p>\n' \
               f'<p class="chapter_url"><a href="{articleUrl}">文章详细地址：{articleUrl}</a></p>\n' \
               f'<p class="Summary"></p>\n' \
               f'<p class="pub_data">{pub_date}</p>\n'
        content = f'<div class="content">{author}{content}</div>'
        content = head + '\n' + content
        return docTitle, content, date

    def save_content(self, path, title, content, date):
        new_date = str(date).replace('/', '-')
        timestamp = str(time.time()).split('.')[1]
        if title == None:
            if content:
                title = timestamp
            else:
                return
        new_title = ''.join(re.findall(r'([\u2E80-\u9FFF0-9a-zA-Z“”：？！《》，-]+)', title))
        if '.' in new_title:
            new_title = new_title.replace('.', '点')
        if 'BR' in new_title:
            new_title = new_title.replace('BR', '')

        path = path.replace('\\', '\\\\')
        filedspath = f'{path}\\{new_date}\\'
        if not os.path.exists(filedspath):
            os.mkdir(filedspath)
        filedsname = filedspath + f'{new_title}.html'
        with open(filedsname, mode='w', encoding='utf-8') as f:
            f.write('<!DOCTYPE html>\n<html>\n')
            f.write('<head><meta charset="UTF-8"></head>\n')
            f.write('<body>\n')
            f.write(f'<h1 align="center">{title}</h1>\n')
            f.write(content)
            f.write('\n</body>\n</html>')


class PicSpider(Spider):
    def parseHomePage(self, htmlSource):
        selector = parsel.Selector(htmlSource)
        divs = selector.css('div.channelList_image_article_container__3rM_q div a')
        articleUrls = []
        for a in divs:
            articleUrl = a.css('::attr(href)').get()
            articleUrls.append(articleUrl)
        return articleUrls


# 头条
def runTouTiao(path):
    homeUrl = 'https://zj.zjol.com.cn/?id=52e5f902cf81d754a434fb50'
    toutiao = Spider(url=homeUrl)
    homePageSource = toutiao.run()
    articleUrls = toutiao.parseHomePage(htmlSource=homePageSource)

    for articleUrl in articleUrls:
        content = toutiao.paseArticlePage(articleUrl)
        if content != None:
            title, content, date = content
            toutiao.save_content(f'{path}\\头条', title, content, date)
            print(title, date, articleUrl)


# 天下
def runTianXia(path):
    homeUrl = 'https://zj.zjol.com.cn/?id=5d4ba90a159bb84750661d51'
    tianxia = Spider(url=homeUrl)
    homePageSource = tianxia.run()
    articleUrls = tianxia.parseHomePage(htmlSource=homePageSource)

    for articleUrl in articleUrls:
        content = tianxia.paseArticlePage(articleUrl)
        if content != None:
            title, content, date = content
            tianxia.save_content(f'{path}\\天下', title, content, date)
            print(title, date, articleUrl)


# 浙江
def runZheJiang(path):
    homeUrl = 'https://zj.zjol.com.cn/?id=5d4ba8cd159bb84750661d50'
    zhejiang = Spider(url=homeUrl)
    homePageSource = zhejiang.run()
    articleUrls = zhejiang.parseHomePage(htmlSource=homePageSource)

    for articleUrl in articleUrls:
        content = zhejiang.paseArticlePage(articleUrl)
        if content != None:
            title, content, date = content
            zhejiang.save_content(f'{path}\\浙江', title, content, date)
            print(title, date, articleUrl)


# 战疫
def runZhanYi(path):
    homeUrl = 'https://zj.zjol.com.cn/?id=5e2e4410b4a13d092b0dc969'
    zhanyi = Spider(url=homeUrl)
    homePageSource = zhanyi.run()
    articleUrls = zhanyi.parseHomePage(htmlSource=homePageSource)

    for articleUrl in articleUrls:
        content = zhanyi.paseArticlePage(articleUrl)
        if content != None:
            title, content, date = content
            zhanyi.save_content(f'{path}\\战疫', title, content, date)
            print(title, date, articleUrl)


# 观点
def runGuanDian(path):
    homeUrl = 'https://zj.zjol.com.cn/?id=584e6ac7e200b2098f871d3a'
    guandian = Spider(url=homeUrl)
    homePageSource = guandian.run()
    articleUrls = guandian.parseHomePage(htmlSource=homePageSource)

    for articleUrl in articleUrls:
        content = guandian.paseArticlePage(articleUrl)
        if content != None:
            title, content, date = content
            guandian.save_content(f'{path}\\观点', title, content, date)
            print(title, date, articleUrl)


# 生活
def runShengHuo(path):
    homeUrl = 'https://zj.zjol.com.cn/?id=5534eb21498e2ca4bf9f3c34'
    shenghuo = Spider(url=homeUrl)
    homePageSource = shenghuo.run()
    articleUrls = shenghuo.parseHomePage(htmlSource=homePageSource)

    for articleUrl in articleUrls:
        content = shenghuo.paseArticlePage(articleUrl)
        if content != None:
            title, content, date = content
            shenghuo.save_content(f'{path}\\生活', title, content, date)
            print(title, date, articleUrl)


# 图片
def runTuPian(path):
    homeUrl = 'https://zj.zjol.com.cn/image-list'
    tupian = PicSpider(url=homeUrl)
    homePageSource = tupian.run()
    articleUrls = tupian.parseHomePage(htmlSource=homePageSource)

    for articleUrl in articleUrls:
        content = tupian.paseArticlePage(articleUrl)
        if content != None:
            title, content, date = content
            tupian.save_content(f'{path}\\图片', title, content, date)
            print(title, date, articleUrl)


# 杭州
def runHangZhou(path):
    homeUrl = 'https://zj.zjol.com.cn/local?id=53845624e4b08e9fb1cdfc17'
    hangzhou = Spider(url=homeUrl)
    homePageSource = hangzhou.run()
    articleUrls = hangzhou.parseHomePage(htmlSource=homePageSource)

    for articleUrl in articleUrls:
        content = hangzhou.paseArticlePage(articleUrl)
        if content != None:
            title, content, date = content
            hangzhou.save_content(f'{path}\\杭州', title, content, date)
            print(title, date, articleUrl)


# 宁波
def runNingBo(path):
    homeUrl = 'https://zj.zjol.com.cn/local?id=53845a6fe4b08e9fb1cdfcac'
    ningbo = Spider(url=homeUrl)
    homePageSource = ningbo.run()
    articleUrls = ningbo.parseHomePage(htmlSource=homePageSource)

    for articleUrl in articleUrls:
        content = ningbo.paseArticlePage(articleUrl)
        if content != None:
            title, content, date = content
            ningbo.save_content(f'{path}\\宁波', title, content, date)
            print(title, date, articleUrl)


# 温州
def runWenZhou(path):
    homeUrl = 'https://zj.zjol.com.cn/local?id=53845aaee4b08e9fb1cdfcb4'
    wenzhou = Spider(url=homeUrl)
    homePageSource = wenzhou.run()
    articleUrls = wenzhou.parseHomePage(htmlSource=homePageSource)

    for articleUrl in articleUrls:
        content = wenzhou.paseArticlePage(articleUrl)
        if content != None:
            title, content, date = content
            wenzhou.save_content(f'{path}\\温州', title, content, date)
            print(title, date, articleUrl)


# 湖州
def runHuZhou(path):
    homeUrl = 'https://zj.zjol.com.cn/local?id=53845b49e4b08e9fb1cdfcc1'
    huzhou = Spider(url=homeUrl)
    homePageSource = huzhou.run()
    articleUrls = huzhou.parseHomePage(htmlSource=homePageSource)

    for articleUrl in articleUrls:
        content = huzhou.paseArticlePage(articleUrl)
        if content != None:
            title, content, date = content
            huzhou.save_content(f'{path}\\湖州', title, content, date)
            print(title, date, articleUrl)


# 嘉兴
def runJiaXing(path):
    homeUrl = 'https://zj.zjol.com.cn/local?id=53845af4e4b08e9fb1cdfcbd'
    jiaxing = Spider(url=homeUrl)
    homePageSource = jiaxing.run()
    articleUrls = jiaxing.parseHomePage(htmlSource=homePageSource)

    for articleUrl in articleUrls:
        content = jiaxing.paseArticlePage(articleUrl)
        if content != None:
            title, content, date = content
            jiaxing.save_content(f'{path}\\嘉兴', title, content, date)
            print(title, date, articleUrl)


# 绍兴
def runShaoXing(path):
    homeUrl = 'https://zj.zjol.com.cn/local?id=53845b81e4b08e9fb1cdfccf'
    shaoxing = Spider(url=homeUrl)
    homePageSource = shaoxing.run()
    articleUrls = shaoxing.parseHomePage(htmlSource=homePageSource)

    for articleUrl in articleUrls:
        content = shaoxing.paseArticlePage(articleUrl)
        if content != None:
            title, content, date = content
            shaoxing.save_content(f'{path}\\绍兴', title, content, date)
            print(title, date, articleUrl)


# 金华
def runJinHua(path):
    homeUrl = 'https://zj.zjol.com.cn/local?id=53845bd9e4b08e9fb1cdfcda'
    jinhua = Spider(url=homeUrl)
    homePageSource = jinhua.run()
    articleUrls = jinhua.parseHomePage(htmlSource=homePageSource)

    for articleUrl in articleUrls:
        content = jinhua.paseArticlePage(articleUrl)
        if content != None:
            title, content, date = content
            jinhua.save_content(f'{path}\\金华', title, content, date)
            print(title, date, articleUrl)


# 衢州
def runQuZhou(path):
    homeUrl = 'https://zj.zjol.com.cn/local?id=53845c2ae4b08e9fb1cdfce3'
    quzhou = Spider(url=homeUrl)
    homePageSource = quzhou.run()
    articleUrls = quzhou.parseHomePage(htmlSource=homePageSource)

    for articleUrl in articleUrls:
        content = quzhou.paseArticlePage(articleUrl)
        if content != None:
            title, content, date = content
            quzhou.save_content(f'{path}\\衢州', title, content, date)
            print(title, date, articleUrl)


# 舟山
def runZhouShan(path):
    homeUrl = 'https://zj.zjol.com.cn/local?id=53845c65e4b08e9fb1cdfce7'
    zhoushan = Spider(url=homeUrl)
    homePageSource = zhoushan.run()
    articleUrls = zhoushan.parseHomePage(htmlSource=homePageSource)

    for articleUrl in articleUrls:
        content = zhoushan.paseArticlePage(articleUrl)
        if content != None:
            title, content, date = content
            zhoushan.save_content(f'{path}\\舟山', title, content, date)
            print(title, date, articleUrl)


# 台州
def runTaiZhou(path):
    homeUrl = 'https://zj.zjol.com.cn/local?id=53845c96e4b08e9fb1cdfcec'
    taizhou = Spider(url=homeUrl)
    homePageSource = taizhou.run()
    articleUrls = taizhou.parseHomePage(htmlSource=homePageSource)

    for articleUrl in articleUrls:
        content = taizhou.paseArticlePage(articleUrl)
        if content != None:
            title, content, date = content
            taizhou.save_content(f'{path}\\台州', title, content, date)
            print(title, date, articleUrl)


# 丽水  来源
def runLiShui(path):
    homeUrl = 'https://zj.zjol.com.cn/local?id=53845cd2e4b08e9fb1cdfcf0'
    lishui = Spider(url=homeUrl)
    homePageSource = lishui.run()
    articleUrls = lishui.parseHomePage(htmlSource=homePageSource)

    for articleUrl in articleUrls:
        content = lishui.paseArticlePage(articleUrl)
        if content != None:
            title, content, date = content
            lishui.save_content(f'{path}\\丽水', title, content, date)
            print(title, date, articleUrl)


# 义乌
def runYiWu(path):
    homeUrl = 'https://zj.zjol.com.cn/local?id=5428f31b498e0d3c0109194e'
    tianxia = Spider(url=homeUrl)
    homePageSource = tianxia.run()
    articleUrls = tianxia.parseHomePage(htmlSource=homePageSource)

    for articleUrl in articleUrls:
        content = tianxia.paseArticlePage(articleUrl)
        if content != None:
            title, content, date = content
            tianxia.save_content(f'{path}\\义乌', title, content, date)
            print(title, date, articleUrl)


# 视频
def runShiPin(path):
    homeUrl = 'https://zj.zjol.com.cn/?id=57d690e7e200b20fbb4af09f'
    shipin = Spider(url=homeUrl)
    homePageSource = shipin.run()
    articleUrls = shipin.parseHomePage(htmlSource=homePageSource)

    for articleUrl in articleUrls:
        content = shipin.paseArticlePage(articleUrl)
        if content != None:
            title, content, date = content
            shipin.save_content(f'{path}\\视频', title, content, date)
            print(title, date, articleUrl)


def runAll():
    column_list = [runTouTiao, runTianXia, runZheJiang, runZhanYi, runGuanDian, runShengHuo, runTuPian, runHangZhou,
                   runNingBo, runWenZhou, runHuZhou, runJiaXing, runShaoXing, runJinHua, runQuZhou, runZhouShan,
                   runTaiZhou, runLiShui, runYiWu, runShiPin]

    with ThreadPoolExecutor(max_workers=5) as t:
        for column in column_list:
            t.submit(column, 'E:\\data\\zjxww')

    # for i in range(10):
    #     time.sleep(1)
    #     print(i)


def start(minutes=0):
    flg = minutes
    schedule.every(minutes).minutes.do(runAll)
    while True:
        if minutes != 0:
            print(f'等待{minutes}分钟')
        else:
            minutes = flg
        schedule.run_pending()

        
        time.sleep(60)
        minutes = minutes - 1


def exists():
    column_zh_list = ['头条', '天下', '浙江', '战疫', '观点', '生活', '图片', '杭州', '宁波', '温州',
                      '湖州', '嘉兴', '绍兴', '金华', '衢州', '舟山', '台州', '丽水', '义乌', '视频', ]

    if not os.path.exists('E:\\data\\zjxww'):
        os.mkdir('E:\\data\\zjxww')
        for i in column_zh_list:
            filedspath = f'E:\\data\\zjxww\\{i}'
            os.mkdir(filedspath)
    else:
        for i in column_zh_list:
            filedspath = f'E:\\data\\zjxww\\{i}'
            if not os.path.exists(filedspath):
                os.mkdir(filedspath)


if __name__ == "__main__":
    if len(sys.argv) == 2:
        exists()
        minutes = int(sys.argv[1])
        start(minutes)
    if len(sys.argv) == 3:
        exists()
        flg = int(sys.argv[2])
        if flg == 1:
            runAll()
        minutes = int(sys.argv[1])
        start(minutes)

    # runHuZhou('E:\\data\\zjxww')