爬取今日头条

import requests, time, json
from urllib.parse import urlencode


# 获取请求参数
def get_page(offset, search_name):
    params = {
        'aid': '24',
        'app_name': 'web_search',
        'offset': offset,
        'format': 'json',
        'keyword': search_name,
        'autoload': 'true',
        'count': '20',
        'cur_tab': "1",
        'en_qc': "1",
        'from': 'search_tab',
        'pd': 'synthesis',
        'timestamp': int(time.time())
    }
    url = "https://www.toutiao.com/api/search/content/?" + urlencode(params)
    headers = {
        "user-agent": "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Mobile Safari/537.36",
        "cookie": "tt_webid=6818010969870059015; SLARDAR_WEB_ID=de0a6fa0-2353-4361-a091-f750c1d039b3; s_v_web_id=verify_k99dx1pd_hDVP0AHW_rkUL_4NpI_BR7H_CaUU49kDWHk1; WEATHER_CITY=%E5%8C%97%E4%BA%AC; tt_webid=6818010969870059015; csrftoken=e39f6a2faa3b2742b880f8939ab121b8; ttcid=3001d5b7eadf4314a152fb7c729be3cd13; __tasessionId=2ejuhcxwv1587452318019; tt_scid=cnsXgjICkjp-H2tEX2xRGjOHHlzEGHSM0xc-YgTdo.D-QsFU8aZ8YPHn0RcildVsfa8a"
    }
    res = requests.get(url, headers=headers)
    if res.status_code == 200:
        return json.loads(res.text)


# 解析网页
def get_data(dic):
    data = dic.get('data')
    if data:
        for result in data:
            if result.get("title") and result.get("source_url"):
                # print(result["title"])
                # print(result['source_url'])
                yield {
                    "title": result["title"],
                    "source_url": result['source_url']
                }


# 主入口
def main(offset, search_name):
    json = get_page(offset, search_name)

    for item in get_data(json):
        print(item)


if __name__ == '__main__':
    for i in range(4):
        page = i * 20
        main(i, "疫情")