[Python] 快速爬取当前城市所有租房网站房源及配置，一目了然

Python爬取当前城市房源信息，以徐州为例

代码效果图请看下方，其他部分请查看附件，一起学习，谢谢

1. # -*- coding: utf-8 -*-
2. """
3. @Time ： 2020/3/18 22:23
4. @Auth ： Suk
5. @File ： 5.小猪短租相关信息.py
6. @IDE ： PyCharm
7. @Motto： Knowing your ignorance is the best way to succeed.
8. @Tips ： 版权所有，转载，转发请注明，如有侵权请联系，谢谢.
9. """
10. # 小猪短租相关信息，包含出租房屋名称、地址、价格、房东、详细链接等信息
11. # 爬取搜索页面信息，爬取5页相关内容,通过获得的详细链接页面，爬取详细页面内容
13. import bs4
14. import requests
15. from bs4 import BeautifulSoup
17. kv = {
18. 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36 Edg/83.0.478.45'
19. }
20. # format='{0:<10}\t{1:{6}<40}\t{2:{6}<10}\t{3:{6}^10}\t{4:{6}<10}\t{5:{6}<10}'
21. format = '{0:<10}\t{1:{5}<40}\t{2:{5}<10}\t{3:{5}^30}\t{4:{5}<10}'
24. def main():
25. global a
26. print('为您找到了{}条信息，您可以根据需要检索相关信息'.format(len(allInitMessage)))
27. print('-------------------------------------------------------------------------')
28. print(
29. '序号 房屋名称 价格 地址 房东 ')
30. for i in allInitMessage:
31. # print(format.format(i[0],i[1],i[2],i[3],i[4],i[5],chr(12288)))
32. print(format.format(i[0], i[1], i[2], i[3], i[4], chr(12288)))
33. a = eval(input('请输入你想了解的房屋序号：'))
34. return a
37. def houseDetails(num):
38. deurl = allInitMessage[num - 1][-1]
39. detailRe = requests.get(deurl, headers=kv)
40. sp = BeautifulSoup(detailRe.text, 'lxml')
41. print('标题：' + sp.title.string.strip())
42. print("价格：" + sp.find('div', class_='fl').text.strip())
43. print("地址：" + sp.find('span', class_='pr5').text.strip())
44. print("房东：" + sp.find('a', class_='lorder_name').text.strip())
45. print('详细信息：')
46. for i in sp.find('ul', class_='house_info clearfix').text.split():
47. if '：' in i:
48. print("\t" + i)
49. elif "宜住" in i:
50. print("\t" + i)
51. print("个性描述：")
52. for i in sp.find('div', class_='box_white clearfix detail_intro_item').text.split():
53. if "个性描述" in i or "查看全部" in i or "收起" in i:
54. continue
55. else:
56. print("\t" + i)
58. print("内部情况：")
59. for i in sp.find('div', class_='box_gray clearfix detail_intro_item').text.split():
60. if "内部情况" in i or "查看全部" in i or "收起" in i:
61. continue
62. else:
63. print("\t" + i)
65. print("交通情况：")
66. for i in sp.findAll('div', class_='info_r')[2].text.split():
67. if "交通信息" in i or "交通情况" in i or "查看全部" in i or "收起" in i:
68. continue
69. else:
70. print("\t" + i)
71. print("周边情况：")
72. for i in sp.findAll('div', class_='info_r')[3].text.split():
73. if "交通信息" in i or "查看全部" in i or "收起" in i:
74. continue
75. else:
76. print("\t" + i)
78. print("配套条件：")
79. print('\t', end="")
80. for i in sp.findAll('div', class_='info_r')[4].children:
81. if type(i) == bs4.element.Tag:
82. if i.ul != None:
83. for j in i.ul.children:
84. if type(j) == bs4.element.Tag:
85. if 'no' in j.get('class')[0]:
86. continue
87. else:
88. print(j.text.strip() + ',', end="")
90. print('\n' + "入住须知：")
91. for i in sp.findAll('div', class_='info_r')[5].text.split():
92. print('\t' + i)
93. print("押金及其他费用")
94. for i in sp.find('div', class_='clause_box').text.split():
95. if "押金及其他费用" in i or "查看全部" in i or "收起" in i or ">" in i:
96. continue
97. elif i[-1] == "：":
98. print("\t" + i, end="")
99. else:
100. print("\t" + i)
101. try:
102. select = input('是否返回主页面(y/n)?')
103. if select in 'yY':
104. main()
105. return True
106. if select in 'nN':
107. print("退出成功！")
108. return False
110. except:
111. print("ERROR！输入错误！")
114. url = 'http://xuzhou.xiaozhu.com/search-duanzufang-p'
115. allhref = []
116. allInitMessage = []
117. for page in range(1, eval(input('当前城市：徐州\t请输入你想检索的页数，共13页：')) + 1):
118. lis = []
119. hrefs = []
120. print('\t正在检索第' + str(page) + '页')
121. re = requests.get(url + str(page) + '-0/')
122. soup = BeautifulSoup(re.text, 'lxml')
123. for i in soup.find('ul', class_='pic_list clearfix list_code').children:
124. if type(i) == bs4.element.Tag:
125. lis.append(i.find("a"))
126. for i in lis:
127. if i != None:
128. hrefs.append(i.get('href'))
129. for i in hrefs:
130. message = []
131. innitMessageget = requests.get(i)
132. innitMessage = BeautifulSoup(innitMessageget.text, 'lxml')
133. message.append(str(len(allInitMessage) + 1) + '.')
134. message.append(innitMessage.title.string)
135. message.append(innitMessage.find('div', class_='fl').text.strip())
136. message.append(innitMessage.find('span', class_='pr5').text.strip())
137. message.append(innitMessage.find('a', class_='lorder_name').text.strip())
138. message.append(i)
139. allInitMessage.append(message)
140. allhref.append(hrefs)
141. print("\n\n\tMission Success!\n\n\n")
142. main()
143. while (True):
144. bolean = houseDetails(a)
145. if bolean == False:
146. exit(0)
147. else:
148. continue