Python爬虫_豆瓣电视剧

import requests
import json
import csv


class DoubantvSpider:
    def __init__(self):
        # self.proxies = {"http":"http://125.123.152.81:3000"}
        self.url = "https://movie.douban.com/j/search_subjects?type=tv&tag=%E5%9B%BD%E4%BA%A7%E5%89%A7&sort=rank&page_limit=20&page_start={}" # 手机模式下国产剧请求网址
        self.headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36"}

    # 发送请求，获得json，转化为字典
    def parse_url(self, url):
        res = requests.get(url, headers=self.headers)
        return json.loads(res.content.decode())

    # 保存数据
    def save(self, dic):
        with open("doubantv.csv", "a", newline='', encoding="utf8") as f:
            for data in dic["subjects"]:
                writer = csv.writer(f, delimiter=',')
                writer.writerow([data["rate"], data["title"], data["url"]])

    # 实现主要逻辑
    def run(self):
        page_num = 0
        while True:
            # 构造url
            url = self.url.format(page_num)
            # 发送请求,获取响应
            dic = self.parse_url(url)
            # 因为动态加载,通过判断每页电视剧数量来确定是不是到了尾页
            if len(dic["subjects"]) < 20:
                break
            self.save(dic)
            page_num += 20
            print("ok")


if __name__ == "__main__":
    doubantv = DoubantvSpider()
    doubantv.run()