Python爬取疫情历史数据

每日日报24	3月24日
所花时间（包括上课）	7小时
代码量（行）	139
博客量（篇）	2
了解到的知识点	Python爬取疫情历史数据

 

 

 

 

 

 

效果图如下：

其中“各省疫情数据.xls”文件中为各个省的疫情历史数据：

 其余文件为各个省所对应的市的疫情历史数据，以“安徽”为例子：

代码如下：

import sys
import datetime
import requests
import xlwt

def getURLContent(url):
    headers = {
        'user-agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Mobile Safari/537.36'
    }
    response = requests.post(url, headers=headers)
    return response.status_code, response

def getCountry(workbook,province, city):
        worksheet=workbook.add_sheet(city)
        print("开始爬取 %s 的 %s的疫情数据... ... " % (province,city))
        url = 'https://api.inews.qq.com/newsqa/v1/query/pubished/daily/list?province=%s&city=%s' % (province,city)
        status_code, data = getURLContent(url)
        if status_code != 200:
            print("%s数据爬取失败,状态码%d" % (province, status_code))
            sys.exit()
        data = data.json()["data"]
        if data == None:
            print("%s数据爬取数据为空" % province)

        worksheet.col(0).width = 128 * 20  # 设置excel中第A列的宽度（方便日期数据展示）

        current_row_index = 0  # 记录当前所写入数据的行号

        # 将列标题写入excel
        for i, str_col in enumerate(['日期', '地点', '确诊人数', '死亡人数', '治愈人数', '确诊新增']):
            worksheet.write(current_row_index, i, str_col)  # 参数对应 行, 列, 值
        current_row_index += 1

        # 往excel中写入日期格式
        style = xlwt.XFStyle()
        style.num_format_str = 'YYYY-MM-DD'

        # 将抓取到的疫情数据写入excel
        for data_i in data:
            worksheet.write(current_row_index, 0, datetime.datetime.strptime('2020.' + data_i['date'], "%Y.%m.%d"),style)
            worksheet.write(current_row_index, 1, data_i['city'])
            worksheet.write(current_row_index, 2, data_i['confirm'])
            worksheet.write(current_row_index, 3, data_i['dead'])
            worksheet.write(current_row_index, 4, data_i['heal'])
            worksheet.write(current_row_index, 5, data_i['confirm_add'])
            current_row_index += 1
        workbook.save('E:\人数采集3.0\%s的疫情数据.xls' % province)

def getProvince(workbook,province):
    worksheet = workbook.add_sheet(province)
    print("开始爬取 %s 疫情数据... ... " % province)
    url = 'https://api.inews.qq.com/newsqa/v1/query/pubished/daily/list?province=%s' % province
    status_code, data = getURLContent(url)

    if status_code != 200:
        print("%s数据爬取失败,状态码%d" % (province, status_code))
        sys.exit()
    data = data.json()["data"]
    if data == None:
        print("%s数据爬取数据为空" % province)

    worksheet.col(0).width = 128 * 20  # 设置excel中第A列的宽度（方便日期数据展示）

    current_row_index = 0  # 记录当前所写入数据的行号

    # 将列标题写入excel
    for i, str_col in enumerate(['日期', '地点', '确诊人数', '死亡人数', '治愈人数', '确诊新增']):
        worksheet.write(current_row_index, i, str_col)  # 参数对应 行, 列, 值
    current_row_index += 1

    # 往excel中写入日期格式
    style = xlwt.XFStyle()
    style.num_format_str = 'YYYY-MM-DD'

    # 将抓取到的疫情数据写入excel
    for data_i in data:
        worksheet.write(current_row_index, 0, datetime.datetime.strptime('2020.' + data_i['date'], "%Y.%m.%d"),
                            style)
        worksheet.write(current_row_index, 1, data_i['province'])
        worksheet.write(current_row_index, 2, data_i['confirm'])
        worksheet.write(current_row_index, 3, data_i['dead'])
        worksheet.write(current_row_index, 4, data_i['heal'])
        worksheet.write(current_row_index, 5, data_i['confirm_add'])
        current_row_index += 1
        workbook.save('E:\人数采集3.0\各省疫情数据.xls')

if __name__ == "__main__":

    # 创建一个workbook 设置编码
    workbook = xlwt.Workbook(encoding='utf-8')
    country = {"北京", "天津", "上海", "重庆", "河北", "山西", "辽宁", "吉林", "黑龙江", "江苏", "浙江", "安徽", "福建", "江西", "山东",  "河南", "湖北", "湖南", "广东", "海南", "四川", "贵州", "云南", "陕西", "甘肃", "青海", "台湾", "新疆", "宁夏", "西藏", "广西", "内蒙古", "香港", "澳门"}
    for count in country:
        getProvince(workbook,count)
        print(count)

    # 添加要爬取疫情数据的国家
    d = {
        '河北': ["石家庄", "唐山", "秦皇岛", "邯郸", "邢台", "保定", "张家口", "承德", "沧州", "廊坊", "衡水"],
        '山西': ["太原", "大同", "阳泉", "长治", "晋城", "朔州", "忻州", "吕梁", "晋中", "临汾", "运城"],
        '辽宁': ["沈阳", "大连", "鞍山", "抚顺", "本溪", "丹东", "锦州", "营口", "阜新", "辽阳", "盘锦", "铁岭", "葫芦岛"],
        '吉林': ["长春", "吉林", "四平", "辽源", "通化", "白城", "松原"],
        '黑龙江': ["哈尔滨", "齐齐哈尔", "牡丹江", "佳木斯", "大庆", "伊春", "鸡西", "鹤岗", "双鸭山", "七台河", "绥化", "黑河"],
        '江苏': ["南京", "无锡", "徐州", "常州", "苏州", "南通", "连云港", "淮安", "盐城", "扬州", "镇江", "泰州", "宿迁"],
        '浙江': ["杭州", "宁波", "温州", "绍兴", "湖州", "嘉兴", "金华", "衢州", "台州", "丽水", "舟山"],
        '安徽': ["合肥", "芜湖", "蚌埠", "淮南", "马鞍山", "淮北", "铜陵", "安庆", "黄山", "阜阳", "宿州", "滁州", "六安", "宣城", "池州", "亳州"],
        '福建': ["福州", "莆田", "泉州", "厦门", "漳州", "龙岩", "三明", "南平", "宁德"],
        '江西': ["南昌", "赣州", "宜春", "吉安", "上饶", "抚州", "九江", "景德镇", "萍乡", "新余", "鹰潭"],
        '山东': ["济南", "青岛", "淄博", "枣庄", "烟台", "潍坊", "济宁", "泰安", "威海", "日照", "滨州", "德州", "聊城", "临沂", "菏泽"],
        '河南': ["郑州", "开封", "洛阳", "平顶山", "安阳", "鹤壁", "新乡", "焦作", "濮阳", "许昌", "漯河", "三门峡", "商丘", "周口", "驻马店", "南阳", "信阳"],
        '湖北': ["武汉", "黄石", "十堰", "荆州", "宜昌", "襄阳", "鄂州", "荆门", "黄冈", "孝感", "咸宁", "随州"],
        '湖南': ["长沙", "株洲", "湘潭", "衡阳", "邵阳", "岳阳", "张家界", "益阳", "常德", "娄底", "郴州", "永州", "怀化"],
        '广东': ["广州", "深圳", "珠海", "汕头", "佛山", "韶关", "湛江", "肇庆", "江门", "茂名", "惠州", "梅州", "汕尾", "河源", "阳江", "清远", "东莞","中山", "潮州", "揭阳"],
        '海南': ["海口", "三亚", "儋州"],
        '四川': ["成都", "绵阳", "自贡", "攀枝花", "泸州", "德阳", "广元", "遂宁", "内江", "乐山", "资阳", "宜宾", "南充", "达州", "雅安", "广安", "巴中","眉山"],
        '贵州': ["贵阳", "六盘水", "遵义", "铜仁", "毕节", "安顺"],
        '云南': ["昆明", "曲靖", "玉溪", "普洱", "临沧"],
        '陕西': ["西安", "铜川", "宝鸡", "咸阳", "渭南", "汉中", "安康", "商洛", "延安", "榆林"],
        '甘肃': ["兰州", "金昌", "白银", "天水", "张掖", "定西", "陇南", "平凉", "庆阳"],
        '青海': ["西宁"],
        '新疆': ["乌鲁木齐", "吐鲁番"],
        '宁夏': ["银川", "石嘴山", "吴忠", "固原", "中卫"],
        '西藏': ["拉萨"],
        '广西': ["南宁", "柳州", "桂林", "梧州", "北海", "来宾", "贺州", "玉林", "百色", "河池", "钦州", "防城港", "贵港"],
        '内蒙古': ["呼和浩特", "包头", "乌海", "赤峰", "呼伦贝尔", "通辽", "乌兰察布", "鄂尔多斯", "巴彦淖尔"],
    }

    def itertransfer(d):
        for k, values in d.items():
            for v in values:
                yield (k, v)

    for name in d.keys():
        print(name.title())
        # 创建一个workbook 设置编码
        workbookk = xlwt.Workbook(encoding='utf-8')
        for k,v in itertransfer(d):
            if k==name.title():
                getCountry(workbookk, k, v)
                print(k,v)