链家网深圳各区租房信息爬取（多线程爬取）

代码：

import requests
from pyquery import PyQuery as pq
import csv
from threading import Thread,Lock
import time
import os

def get_one_url(url):

    try:
        headers={
            'Referer':'https: // s1.ljcdn.com / matrix_pc / dist / pc / src / common / css / common.css?_v = 20191213202326259',
            'User - Agent':'Mozilla / 5.0(Windows NT 10.0;Win64;x64) AppleWebKit / 537.36(KHTML, likeGecko) Chrome / 76.0.3809.132Safari / 537.36',

        }
        # 获取响应
        response=requests.get(url,headers,proxies=None)
        # 判断请求状态
        if response.status_code==200:
            return response.text
        return None
    except requests.exceptions.ConnectionError as e:
        print('error',e.args)

def parse_one_url(html):
    doc=pq(html)
    items=doc('.content__list--item--main').items()
    for item in items:
        # 上锁,等待每个线程的解析执行完之后在执行其他线程
        mutes.acquire()
        info= {
            'content': item.find('a').text().split()[0],
            'room': item.find('a').text().split()[1],
            'direction': item.find('a').text().split()[2],
            'size':item.find('p.content__list--item--des').text().split('/')[1],
            'price':item.find('.content__list--item-price').text(),
            'location':item.find('.content__list--item--des a').text().replace(' ','-')
        }
        # 释放互斥锁
        mutes.release()
        print(info)
        # 生成器，使循环结束之后再返回
        yield info

# 创建互斥锁
mutes = Lock()

def man(page,city):
    url = 'https://sz.lianjia.com/zufang/%s/pg%d/#contentList'%(city,page)
    html = get_one_url(url)
    infos=parse_one_url(html)
    save(infos,city)


def save(infos,city):
    '''
    判断文件是否存在，如果存在就说明表头已经写了就不写表头，反之加上表头
    :param infos: 保存的信息
    :return:
    '''
    if os.path.exists('%s.csv' %(city)):
        with open('链家租房.csv', 'a', encoding='utf-8') as csvfile:
            fieldname = ['content', 'room', 'direction', 'size', 'location', 'price']
            writer = csv.DictWriter(csvfile, fieldname)
            for info in infos:
                writer.writerow(info)
    else:
        with open('%s.csv' %(city),'a',encoding='utf-8') as csvfile:
            fieldname=['content','room','direction','size','location','price']
            writer = csv.DictWriter(csvfile,fieldname) #DictWriter方法使csv文件可以写入字典
            writer.writeheader()
            for info in infos:
                writer.writerow(info)


if __name__=='__main__':
    cities = ['luohuqu', 'longhuaqu', 'futianqu']
    # 循环深圳各区的缩写
    for city in cities:
        ts =[]
        # 创建多个线程分页爬取，使爬虫效率大大提高
        for i in range(1,50):
            exec('t{0} = Thread(target=man,args=(i,city))'.format(i))
            exec('ts.append(t{0})'.format(i))
        for t in ts:
            t.start()
        # 等待所有线程全部执行结束
        time.sleep(4)
        print('%s租房信息存入成功' %(city))
    print('全部存入成功')

　　改进：可以动态爬取，用户输入想要爬取的深圳区域，程序自动爬取并保存；可以多进程爬取，同时爬取不同区域的住房信息并分别保存不同的csv文件；如果怕被封ip,还可以连接代理的 api,用代理爬取。