9.re获取猫眼电影top300

"""
需求：爬取猫眼电影top300
数据来源：https://piaofang.maoyan.com/rankings/year
"""
import csv
import re
import requests

"""
正则re匹配思路：不管页面标签层级，只看"特征锚定"。
"""
def get_html(url):
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
    }
    response = requests.get(url=url,headers=headers,verify=False)
    # print(response.text)
    text = response.text
    pattern = re.compile(r'<ul class=".*?" data-com=".*?">.*?<li class=".*?">(?P<rank>.*?)</li>'
                            r'.*?<p class="first-line">(?P<name>.*?)</p>'
                            r'.*?<p class="second-line">(?P<date>.*?)</p>'
                            r'.*?<li class="col2 tr">(?P<ticket>.*?)</li>'
                            r'.*?<li class="col3 tr">(?P<avg_ticket>.*?)</li>',re.S)
    lis = pattern.finditer(text)
    # print(list(lis))
    for li in lis:
        # print(li.group("rank"))
        # print(li.group("name"))
        # print(li.group("date"))
        # print(li.group("ticket"))
        # print(li.group("avg_ticket"))
        data_dict = li.groupdict()  # re.groupdict() 必须配合re+(?P<>)分组捕获使用
        data_dict["ticket"] = data_dict["ticket"] + "万元"   # 票房拼接单位万元
        data_dict["avg_ticket"] = data_dict["avg_ticket"] + "元"   # 票房拼接单位元
        # print(data_dict)
        csv_writer.writerow(data_dict)

def save_csv():
    f = open(r'save_csv\maoyantop300.csv',"w",encoding="utf-8-sig",newline="")
    global csv_writer
    csv_writer = csv.DictWriter(f,['rank','name','date','ticket','avg_ticket'])
    csv_writer.writeheader()
    return f


def main():
    url = 'https://piaofang.maoyan.com/rankings/year'
    file = save_csv()
    get_html(url)
    file.close()
    print("over!")

if __name__ == '__main__':
    main()