(1)cctv-获取内容

内容解析
目录结果

#weixin_wechat/xinwenlianbo/get_cctv_xinwenlianbo.py
# -*- coding: utf-8 -*-
"""
爬虫源：[新闻联播](https://tv.cctv.com/lm/xwlb/index.shtml?spm=C31267.PFsKSaKh6QQC.EYAqX3oMSyve.3 )]
"""

import requests 
import re
import time
#import pprint
#from bs4 import BeautifulSoup
from datetime import datetime ,timedelta
import json
from lxml import etree

# 輸入結果列表
def get_html_loop(url,loop=2):
    response_url = None
    statuscode = 0
    response_html = None
    if url is None or len(url) ==0:
        print('one invalid url is found!')
        return response_html
    print('url:'+url)
    for i in range(loop):
        try:
            response = requests.get(url)
            response.encoding = 'utf-8'
            if response.status_code == 200:
                response_html=response.text
                response_url = response.url
                statuscode = response.status_code
            else:
                print('response from %s is invalid!' % url)
            break
        except:
            print("Connection refused by the server..")
            print("Let me sleep for 5 seconds")
            print("ZZzzzz...")
            time.sleep(1)
            print("Was a nice sleep, now let me continue...")
            continue
    return response_html
# url = "https://tv.cctv.com/lm/xwlb/day/20210222.shtml"
# response_html = get_html_loop(url)
# print(response_html)


def get_content(response_html):
    html =  etree.HTML(response_html)
    result_title = html.xpath('//div[@class="title"]//text()')
    items_href = html.xpath('//li/a/@href')
    return result_title,items_href


#title_list, items_href= get_content(response_html)


def handle_toc(title_list):
    title_list =  [ value.replace('[视频]','') for value in  title_list]
    title_toc = [ str(title_list.index(value)+1)+'.'+value for value in  title_list]
    title_toc_html =  '<ul style="list-style-type:none;">\n' + '\n'.join(['<li>'.rjust(8) + name + '</li>' for name in title_toc]) + '\n</ul>'
    return title_toc_html
    pass



# def handle_href_toc(href_toc):
#     response_html = get_html_loop(href_toc,loop=2)
#     html =  etree.HTML(response_html)
#     content =html.xpath('//*[@id="chbox01"]/div[2]/div[1]/div/div[1]/p[3]//text()')
#     """
#     标题时间直接使用基础url的title拼接
#     """
#     content = handle_string(str(content))
#     content = content.replace('视频简介：','').replace('本期节目主要内容：','今天央视新闻联播直播主要内容新鲜事：')
#     #content = content.replace('视频简介：本期节目主要内容：\\r\\n','今天央视新闻联播直播主要内容新鲜事：').replace("'',",'')
#     print(content)
#     items = content.split('\\r\\n')
#     print('列表长度',len(items))
#     if len(items) >3 :
#       items = items
#     elif len(items) <3:
#       items = content.splitlines()
#     else :
#       items = re.split(r"[~\r\n]+", content)
#     items_to_html = '<ul style="list-style-type:none;">\n' + '\n'.join(['<li>'.rjust(8) + name + '</li>' for name in items]) + '\n</ul>'
#     return items_to_html

def handle_href_toc(href_toc):
    response_html = get_html_loop(href_toc,loop=2)
    html =  etree.HTML(response_html)
    content =html.xpath('//*[@id="chbox01"]/div[2]/div[1]/div/div[1]/p[3]//text()')
    """
    标题时间直接使用基础url的title拼接
    """
    content = handle_string(str(content))
    content = content.replace('视频简介：,','').replace('本期节目主要内容：','今天央视新闻联播直播主要内容新鲜事：')
    #content = content.replace('视频简介：本期节目主要内容：\\r\\n','今天央视新闻联播直播主要内容新鲜事：').replace("'',",'')
    print(content)
    content = re.sub('\s+', '', str(content)).strip()
    content = re.sub(r'\\r\\n','',content).strip()
    content = re.sub(r"'',",'',content).strip()
    content = re.sub(r"','",'',content).strip() 
    content = content.replace(' ','')
    items = content.split('；')
    print('列表长度',len(items))
    items_to_html = '<ul style="list-style-type:none;">\n' + '\n'.join(['<li>'.rjust(8) + name + '</li>' for name in items]) + '\n</ul>'
    return items_to_html



#title_toc_html = handle_toc(title_list)


def handle_href(href):
    response_html = get_html_loop(href,loop=2)
    html =  etree.HTML(response_html)
    href_title = html.xpath('//div[@class="cnt_nav"]/h3//text()')
    href_title = handle_string(str(href_title))
    href_content = html.xpath('//div[@class="mbd"]/div[@class="cnt_bd"]//text()')
    href_content = handle_string(str(href_content))
    href_source =  html.xpath('//div[@class="mtab_con"]/div[@class="chblock"]//text()')
    href_source = handle_string(str(href_source)).replace('\\r\\n','').replace('\\t','').replace(',','')
    href_pic = html.xpath('//meta[@property="og:image"]/@content')
    href_pic = handle_string(str(href_pic)).replace('//','')
    return href_title,href_content,href_source,href_pic



def handle_href_html(index,href):
    response_html = get_html_loop(href,loop=2)
    html =  etree.HTML(response_html)
    """
    标题处理
    """
    href_title = html.xpath('//div[@class="cnt_nav"]/h3//text()')
    href_title = handle_string(str(href_title))
    href_title =handle_h1_style(index,href_title) 
    """
    内容处理
    """
    href_content = html.xpath('//div[@class="mbd"]/div[@class="cnt_bd"]')[0]
    href_content = etree.tostring(href_content, pretty_print = True).decode()
    ## print(etree.tostring(href_content,pretty_print=True,encoding='unicode'))
    #-------
    href_source =  html.xpath('//div[@class="mtab_con"]/div[@class="chblock"]//text()')
    href_source = handle_string(str(href_source)).replace('\\r\\n','').replace('\\t','').replace(',','')
    #----
    href_pic = html.xpath('//meta[@property="og:image"]/@content')
    href_pic = handle_string(str(href_pic)).replace('//','')
    return href_title,href_content,href_source,href_pic


def handle_string(text):
    """
    处理视频、[]
    """
    text = text.replace('[视频]','').replace('[','').replace(']','')
    """
    处理''
    """
    text = text.replace("'",'')
    return text


def handle_h1_style(index,text):
    text = "<p><h3><strong>" + str(index)+'.'+text + "</strong></h3></p>"
    return text


def get_url(string_date):
    base_url = 'https://tv.cctv.com/lm/xwlb/day/'
    url = base_url+string_date+'.shtml'
    return url



def get_cctv_content():
    today = datetime.today() 
    today_weekday_name = today.strftime('%A')
    yesterday  = datetime.today() - timedelta(days=1)
    yesterday = yesterday.strftime('%Y%m%d')
    string_date = yesterday
    url = get_url(string_date)
    #url = "https://tv.cctv.com/lm/xwlb/day/20210222.shtml"
    response_html = get_html_loop(url)
    title_list, items_href= get_content(response_html)
    title_toc_html = handle_toc(title_list)
    #--
    #--
    href = items_href[1]
    href_title,href_content,href_source,href_pic = handle_href_html(href)
    #--
    print(href_title)
    return title_toc_html

if __name__ == "__main__":
    start_time = time.time() # 开始时间
    title_toc_html = get_cctv_content() 
    print(title_toc_html)
    end_time = time.time() #结束时间
    print("程序耗时%f秒." % (end_time - start_time))