scrapy 解决302问题

self.meta = {
        'dont_redirect': True,  # 禁止网页重定向
        'handle_httpstatus_list': [301, 302]  # 对哪些异常返回进行处理
    }
def start_requests(self):
        sql="select * from lpwz where wz not in (select wz from xinxi1)"
        #self.get_one()
        i=0
        self.con.execute(sql)
        while 1:
            #self.con.execute(sql)
            urlss=self.con.fetchone()
            print("rrrrrrrrrrrrrrrrrrrrrr",urlss[0],type(urlss))
            yield Request(urlss[0],meta=self.meta,callback=self.parse)

 

posted @ 2021-11-20 09:54  myrj  阅读(236)  评论(0)    收藏  举报