python爬取上海10年天气数据

通过python爬取上海10年天气数据

爬取网页 http://tianqi.2345.com/wea_history/58362.htm

爬取上海2011-2020年天气数据

url = "http://tianqi.2345.com/Pc/GetHistory"

headers = {
"User-Agent": """Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36"""
}

import requests
import pandas as pd

def craw_table(year, month):
    params = {
        "areaInfo[areaId]": 58362,
        "areaInfo[areaType]": 2,
        "date[year]": year,
        "date[month]": month
    }

    resp = requests.get(url, headers=headers, params=params)
    data = resp.json()["data"]
    df = pd.read_html(data)[0]
    return df


df_list = []
for year in range(2011,2020):
    for month in range(1,13):
        print("爬取：",year,month)
        df = craw_table(year,month)
        df_list.append(df)

pd.concat(df_list).to_excel("上海2011-2020年天气数据.xlsx", index=False)