今日头条

import requests
import re
import json
import os
from urllib import request
for i in range(0,60,20):
    url = 'https://www.toutiao.com/search_content/?offset={}&format=json&keyword=%E8%A1%97%E6%8B%8D&autoload=true&count=20&cur_tab=1&from=search_tab'.format(i)
    response = requests.get(url)

    # 可以通过response.json 直接获取转化后的对象(dict)
    html_json_dict = response.json()
    #print(html_json_dict)

    # 获取dict中的data key对应的列表
    data_list = html_json_dict['data']

    # 如果列表中的每一项,有article_url我们就取这个值
    for data_item in data_list:
        if 'article_url' in data_item:
            article_url = data_item['article_url']
            headers = {
                'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.26 Safari/537.36 Core/1.63.5733.400 QQBrowser/10.2.2050.400'
            }
            response = requests.get(article_url, headers=headers)
            # 然后就是之前那段代码
            html_str = response.text
            pattern = r'gallery: JSON\.parse\((.*)\),'
            match_res = re.search(pattern, html_str)

            # 新建文件夹
            if not os.path.exists('jiepaiss'):
                os.mkdir('jiepaiss')

            if match_res:
                # print(match_res.group(1))
                # print(type(match_res.group(1)))
                json_origin = match_res.group(1)
                res_str = json.loads(json_origin)
                # print(type(res_str))
                res_dict = json.loads(res_str)
                # print(type(res_dict))

                sub_images_list = res_dict['sub_images']
                for image in sub_images_list:
                    image_url = image['url']
                    filename = 'jiepaiss/' + image_url.split('/')[-1] + '.jpg'
                    print(filename)
                    # 下载图片
                    request.urlretrieve(image_url, filename)
            else:

　　　　　　　　　　print('你写错了, 不应该来我这')