一点资讯 视频抓取 phantomjs

# _*_ coding: utf-8 _*_

"""
思路:
1.列表页使用phantomjs模拟点击2.每个链接只抓取第一页9-10条内容,按照标题去重
3.布置定时任务,每天8点执行一次
"""

import MySQLdb
import redis
import sys
import os
import re
import urllib
import requests
import time
import hashlib
import traceback
import urlparse
import random
import signal
# import multiprocessing
import matplotlib
matplotlib.use("Agg")
import shutil
import socket #图片下载延迟的
socket.setdefaulttimeout(30)
import multiprocessing
from config import IConfig
from video_list import ydzx_url_list
from bs4 import BeautifulSoup
from upload_images import UploadFile
from moviepy.editor import VideoFileClip
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities

reload(sys)
sys.setdefaultencoding('utf-8')

class WxpnVideo(multiprocessing.Process):

    def __init__(self):
        self.redisConf = IConfig.load('resource.redis')
        self.redisServer = redis.Redis(host=self.redisConf['host'], port=self.redisConf['port'], db=self.redisConf['db'], password=self.redisConf['passwd'])

        self.dbConfig = IConfig.load('resource.mysql')
        self.conn = MySQLdb.connect(
            user = self.dbConfig['user'],
            passwd = self.dbConfig['password'],
            db = self.dbConfig['dbname'],
            host = self.dbConfig['host'],
            charset = "utf8",
            use_unicode = True)

        self.conn.ping(True)
        self.cursor = self.conn.cursor()

        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36',
            'Host': 'www.yidianzixun.com',
            'X-Requested-With': 'XMLHttpRequest',
            'Connection': 'keep-alive',
            'Accept': '*/*',
            'Accept-Encoding': 'gzip, deflate',
            'Accept-Language': 'zh-CN,zh;q=0.9',
        }

        self.domain = IConfig.load('resource.domain')
        self.apiConf = IConfig.load('resource.apiurl')

        self.key_video_list = 'wxpn:video:list'
        self.key_title = 'wxpn:video:title'

        self.storeConfig = IConfig.load('resource.store')
        self.thumb_path = self.storeConfig['images_path']

        self.ossConf = IConfig.load('resource.oss')
        self.key_id = self.ossConf['access_key_id']
        self.key_secret = self.ossConf['access_key_secret']
        self.endponit = self.ossConf['endponit']

        self.img_upload = UploadFile()
        self.auth = self.img_upload.auth_oss(self.key_id, self.key_secret)

        self.videoConf = IConfig.load('resource.apiurl')
        self.video_publish = self.videoConf['video_publish_api']
        self.ydzx_page_api = self.videoConf['ydzx_page_api']

        self.start_time = int(time.time())
        multiprocessing.Process.__init__(self)

    def store_video_list_redis(self, video_list):

        if video_list:
            for per_list in video_list:
                if not self.redisServer.sismember(self.key_video_list, per_list):
                    self.redisServer.sadd(self.key_video_list, per_list)
        else:
            return False

    def get_video_para(self):
        while True:
            if self.redisServer.scard(self.key_video_list) == 0:
                break

            link = self.redisServer.spop(self.key_video_list)
            print(link)
            # url = self.ydzx_page_api + link

            # try:
            #     res = requests.get(url=url, timeout=60)
            # except Exception as e:
            #     print('连接失败')
            # print(res.status_code)

            dcap = dict(DesiredCapabilities.PHANTOMJS)
            dcap["phantomjs.page.settings.userAgent"] = (
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36"
            )
            try:
                driver = webdriver.PhantomJS(desired_capabilities=dcap, executable_path='/usr/local/phantomjs/bin/phantomjs')

                # driver.set_page_load_timeout(10)
                # driver.set_script_timeout(10)

                time.sleep(random.randrange(3, 8))
                driver.get(link)
                time.sleep(random.randrange(2, 6))

                text = driver.page_source

                driver.service.process.send_signal(signal.SIGTERM)
                driver.quit()

            except Exception as e:

                print(traceback.format_exc())
                continue

            # if res.status_code == 200:
            soup = BeautifulSoup(text, 'lxml')
            title_list = soup.select('div.channel-news div.doc-title')
            itemid_list = soup.select('div.channel-news a.style-content-middle')

            if title_list and itemid_list:

                try:
                    for num, title in enumerate(title_list):
                        m = hashlib.md5()
                        m.update(str(title.text).strip())
                        psw = m.hexdigest()

                        print(title.text)
                        itemid = itemid_list[num]['data-docid']

                        if not self.redisServer.sismember(self.key_title, psw):

                            yield psw, itemid
                except Exception as e:
                    print(traceback.format_exc())
                    continue

            else:
                print('一点资讯视频列表{0}页链接请求失败,请及时查看原因'.format(link))

                # self.form_data['content'] = '【创业黑马】预警:一点资讯视频列表{0}页链接请求失败,请及时查看原因'.format(link)
                # res = requests.post(self.msg_api, data=self.form_data)

    def time_cycle(self,origin_time):
        now = time.time()

        try:
            if origin_time == '昨天':
                published = int(now) - 24*3600
            elif '天' in origin_time:
                day_one = re.compile('(.*?)天')
                published = int(now)-int(day_one.findall(origin_time)[0])*24*3600
            elif '小时' in origin_time:
                hour_one = re.compile('(.*?)小时')
                published = int(now)-int(hour_one.findall(origin_time)[0])*24*60
            elif '分' in origin_time:
                min_one = re.compile('(.*?)分')
                published = int(now)-int(min_one.findall(origin_time)[0])*60
            elif '月' in origin_time:
                month_one = re.compile('(.*?)个月')
                published = int(now)-int(month_one.findall(origin_time)[0])*30*24*3600
            else:
                timeArray = time.strptime(origin_time, "%Y.%m.%d")
                published = int(time.mktime(timeArray))
            return published
        except Exception as e:
            print(traceback.format_exc())

    def download_video(self, psw, itemid):
        now = int(time.time())

        url = 'http://www.yidianzixun.com/article/' + itemid
        print(url)
        self.headers['Referer'] = url
        try:
            res = requests.get(url=url, headers=self.headers, timeout=60)
            print(res.status_code)
        except Exception as e:
            print('小链接连接失败')

        if res.status_code == 200:
            soup = BeautifulSoup(res.text, 'lxml')

            title = soup.select('div.left-wrapper > h2')[0].text

            try:
                video_src = soup.select('div.video-wrapper > video')[0]['src']
            except Exception as e:
                print('此篇为文章,不是视频')

            thumb_src = soup.select('div.video-wrapper > video')[0]['poster']

            try:
                source = soup.select('body.page-article .left-wrapper > .meta > a')[0].text
            except Exception as e:
                source = soup.select('body.page-article .left-wrapper > .meta > span')[0].text
                source_re = re.sub('来源：', '', str(source))
                source = source_re

            publishtime = soup.select('body.page-article .left-wrapper > .meta > span')[0].text

            try:
                timestamp = self.time_cycle(str(publishtime))
            except Exception as e:
                timestamp = now

            img_url_parts = urlparse.urlparse(thumb_src)
            img_url_query = urlparse.parse_qs(img_url_parts.query,True)

            if img_url_query.has_key('wx_fmt'):
                ext_name = '.' + img_url_query['wx_fmt'][0]
            else:
                ext_name = '.png'

            thumb_p = self.thumb_path + 'video/thumb'
            if not os.path.exists(thumb_p):
                os.mkdir(thumb_p)

            img_down_local_path = thumb_p + '/' + psw[:20] + ext_name
            urllib.urlretrieve(thumb_src, img_down_local_path)
            file_name = psw[:20] + ext_name

            if os.path.exists(img_down_local_path):
                images_path = self.ossConf['video_thumb_path']
                status = self.img_upload.upload_to_oss(self.auth, self.endponit, images_path, file_name, img_down_local_path)

            thumb_src = self.domain['img_url_oss'] + 'Cmstop/ydzx/' + file_name
            m = hashlib.md5()
            m.update(str(thumb_src))
            psw_thumb = m.hexdigest()

            try:
                delay_re = re.compile('"duration":(\d+)')
                playtime = delay_re.findall(str(res.text))[0]
            except Exception as e:
                print(traceback.format_exc())
                playtime = None

            video_path = self.thumb_path + 'video/' + str(video_src).split('/')[-1]

            video_res = requests.get(video_src).content
            with open(video_path, 'wb') as f:
                f.write(video_res)

            try:
                clip = VideoFileClip(video_path)
                print(clip.duration)
            except Exception as e:
                print(traceback.format_exc())
                return False

            # with open(video_path, 'r') as f:
            #     length = len(f.read())
            #     if length < 819200:
            #         return False

            video_name = str(video_src).split('/')[-1][10:]
            if os.path.exists(video_path):
                images_path = self.ossConf['video_path']
                status = self.img_upload.upload_to_oss(self.auth, self.endponit, images_path, video_name, video_path)

                print('')
                if status != 'success':
                    return False

            print('视频上传成功')
            video_link = self.domain['img_url_oss'] + 'Cmstop/video/ydzx/' + video_name

            try:
                sql = "insert into cmstop_comment_topic(title, description, thumb, created, url_md5, url) values(%s, '', %s, %s, %s, '')"
                params = (title, thumb_src, now, psw_thumb)
                self.cursor.execute(sql, params)
                self.conn.commit()

                topicid = self.cursor.lastrowid

            except Exception as e:
                print(traceback.format_exc())
                self.conn.rollback()

            sourceid = self.get_article_sourceid(source)

            try:
                result = self.cursor.execute("""
                            insert into cmstop_content(topicid, sourceid, catid, modelid, title, subtitle, source_title, source_link, weight, status, created, score, published, thumb, createdby)
                            values(%s, %s, %s, %s, %s, %s, %s, %s, 60, %s, %s, %s, %s, %s, %s)
                            """, (topicid, sourceid, 47, 4, title, None, title, '', 3, now, 0, timestamp, thumb_src, 0))
                self.conn.commit()

                lastrowid = self.cursor.lastrowid
            except Exception as e:
                print(traceback.format_exc())
                self.conn.rollback()

            video_id = str(video_src).split('/')[-1][10:-4]
            try:
                sql = "insert into cmstop_video(contentid, video, playtime, author, video_id, aid) values(%s, %s, %s, %s, %s, %s)"
                print(sql)
                params = (lastrowid, video_link, playtime, source, video_id, 0)
                self.cursor.execute(sql, params)
                self.conn.commit()

            except Exception as e:
                print(traceback.format_exc())
                self.conn.rollback()

            self.redisServer.sadd(self.key_title, psw)

            api_url = self.video_publish + str(lastrowid)
            try:
                resp = urllib.urlopen(api_url)
                result = resp.read()
            except:
                print 'connect failed'

        else:
            print('一点资讯视频主链接请求失败,请及时查看原因')

            # self.form_data['content'] = '【创业黑马】预警:一点资讯视频主链接请求失败,请及时查看原因'
            # res = requests.post(self.msg_api, data=self.form_data)

    def get_article_sourceid(self, source, medias = []):
        source = source.strip()
        sourceid = 0

        """
        print source
        print set([source.encode('utf-8')])
        print medias
        """

        result = self.cursor.execute('select `sourceid`, `name`, `has_signed_contract` from `cmstop_source` where `name`="' + source + '"')
        has_signed_contract = 0

        if medias and (set([source.encode('utf-8')]) & medias):
            has_signed_contract = 1

        if result:
            data = self.cursor.fetchone()
            sourceid = data[0]

            if data[2] != has_signed_contract:
                try:
                    result = self.cursor.execute("""
                        update `cmstop_source` set `has_signed_contract`=%s where sourceid=%s
                        """, (has_signed_contract, sourceid))
                    self.conn.commit()
                except:
                    self.conn.rollback()
        else:
            try:
                result = self.cursor.execute("""
                    insert into `cmstop_source`(`name`, `logo`, `url`, `initial`, `has_signed_contract`)
                    values(%s, %s, %s, %s, %s)
                    """, (source, '', '', '', has_signed_contract))
                self.conn.commit()
                sourceid = self.cursor.lastrowid
            except:
                self.conn.rollback()

        return sourceid

    def run(self):
        os.system('pkill phantomjs')

        lockConf = IConfig.load('resource.lock')
        lock_file = lockConf['lock_path_ydzx']

        if os.path.exists(lock_file):
            print('lock file exists')
            return False
        os.system(r'touch %s '% lock_file)

        self.store_video_list_redis(ydzx_url_list)
        get_video_para = self.get_video_para()

        for psw, itemid in get_video_para:

            print(psw)

            stop_time = int(time.time())
            balance_time = stop_time - self.start_time

            if balance_time >= 10800: #运行时间为3个小时

                self.del_file(self.thumb_path + 'video')
                os.system(r'rm -rf %s' % lock_file)
                os._exit(0)

            try:
                self.download_video(psw=psw, itemid=itemid)

                time.sleep(random.uniform(2, 8))

                os.system('pkill ffmpeg-osx-v3.2.4')

            except Exception as e:
                print(traceback.format_exc())
                continue

        self.del_file(self.thumb_path + 'video')
        os.system(r'rm -rf %s' % lock_file)

    def video_publish(self):
        sql = 'select contentid from cmstop_video where contentid<=3528920 and contentid>=3430851'
        self.cursor.execute(sql)
        data = self.cursor.fetchall()

        for num in data:
            api_url = self.video_publish + str(num[0])
            try:
                resp = urllib.urlopen(api_url)
                result = resp.read()
            except:
                print 'connect failed'

    def del_file(self, path):
        os.chdir(path) #进入要清空的目录
        ds = list(os.listdir(path)) #获得该目录下所有文件或文件夹列表皮

        for d in ds: #遍历该列表

            if os.path.isfile(d): #如果列表项是文件
                os.remove(d) #直接删除
            else: #如果不会文件
                shutil.rmtree(d) #也直接删除

if __name__ == '__main__':

    video_one = WxpnVideo()
    # video_two = WxpnVideo()

    video_one.start()
    # video_two.start()

    video_one.join()
    # video_two.join()