Python开发网络爬虫抓取某同城房价信息
前言：
苦逼的我从某某城市换到另一个稍微大点的某某城市，面临的第一个问题就是买房，奋斗10多年，又回到起点，废话就不多说了，看看如何设计程序把某同城上的房价数据抓取过来。
方案：方案思路很简单，先把网页内容获取下来，通过一定规则对内容解析，保存成想要的格式
难点是对网页的解析，是一个比较细致的活，必须边输出，边调试。
具体实现：
获取网页内容：
def get_page(url):
headers = {
'User-Agent': r'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
r'Chrome/45.0.2454.85 Safari/537.36 115Browser/6.0.3',
'Referer': r'http://jn.58.com/ershoufang/',
'Host': r'jn.58.com',
'Connection': 'keep-alive'
}
timeout = 60
socket.setdefaulttimeout(timeout) # 设置超时
req = request.Request(url, headers=headers)
response = request.urlopen(req).read()
page = response.decode('utf-8','ignore')
return page
第二步解析网页：解析时要注意无效内容的处理，不然跑起来会报错，调试很麻烦
def get_58_house(url):
html = get_page(url)
soup = BeautifulSoup(html,"lxml")
table =soup.find(id="main")
df = pd.DataFrame(columns=["op_time","web","house_name","xq","xq1","price","per_price","room","m2","href","ts"])
for tr in table.find_all('tr'):
try:
str_name = tr.find("p","bthead").find("a","t").string.strip()
str_link = tr.find("p","bthead").find("a","t")["href"]
##房产小区位置
str_xq = list()
str_xq1= ''
str_xq2= ''
try:
for s in tr.find_all("a","a_xq1") :
str_xq.append(s.string.strip())
str_xq1= str_xq[0]
str_xq2= str_xq[1]
except:
pass
##房产特色
str_ts =list()
try:
for s in tr.find("div","qj-listleft").stripped_strings:
str_ts.append(s)
except:
pass
## 价格信息####################
str_price =list()
str_toal =''
str_per =''
str_room =''
str_m2 =''
try:
for s in tr.find("div","qj-listright btall").stripped_strings:
str_price.append(s)
str_toal = str_price[0]
str_per = re.findall(r"(\d+\.*\d+)",str_price[1])
str_room = str_price[2]
str_m2 = re.findall(r"(\d+\.*\d+)",str_price[3])
except:
pass
except Exception as e:
print('Exception',":",e)
try:
row = {'web':'58同城','house_name':str_name,'xq':str_xq1,'xq1':str_xq2,'price':str_toal,'per_price':str_per,'room':str_room,'m2':str_m2,'ts':''.join(str_ts),'href':str_link}
newrow = pd.DataFrame(data=row,index=["0"])
df=df.append(newrow,ignore_index=True)
except Exception as e:
print('Exception',":",e)
f=open("log.txt",'a')
traceback.print_exc(file=f)
f.write(row)
f.flush()
f.close()
df["op_time"]=time.strftime('%Y-%m-%d',time.localtime(time.time()))
return df
第三步循环处理每页数据并保存数据：
def get_58_house_all():
##建立数据库连接
engine = create_engine('oracle+cx_oracle://user:password@localhost/orcl')
cnx = engine.connect()
##先清除今天的数据
'''
strSql = 'delete from house where op_time=\'{}\' '.format(time.strftime('%Y-%m-%d',time.localtime(time.time())))
cnx.execute(strSql)
'''
##获取首页房产数据
str_http = "http://jn.58.com/ershoufang/"
writelog(time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))+' Start:'+str_http)
df1=get_58_house(str_http)
try:
df1.to_sql('house', cnx,if_exists='append')
except Exception as e:
'''记录异常信息
本例使用的是oracle 数据库，默认编码格式为GBK，保存时因为特殊字符，导致保存错误。错误提示如下，需要调整oracle字符集
oracle 字符集调整为UTF8,
NLS_LANG: AMERICAN_AMERICA.AL32UTF8
NLS_CHARACTERSET: UTF8
NLS_NCHAR_CHARACTERSET: UTF8
报错信息为
UnicodeEncodeError: 'gbk' codec can't encode character '\xb2' in position 13: illegal multibyte sequence
该字符为上标2，平方米
'''
writelog(time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))+' Except:'+str_http)
df1.to_csv('record.csv',sep=',', encoding='utf-8')
writelog(traceback.format_exc())
writelog(time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))+' End:'+str_http)
time.sleep(20)
##获取其余69页房产数据
for i in range(2,70+1) :
try:
str_http ="http://jn.58.com/ershoufang/pn"+str(i)
writelog(time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))+' Start:'+str_http)
df1=get_58_house(str_http)
df1.to_sql('house', cnx,if_exists='append')
except Exception as e:
##writelog(''.format('Save to database Exception',":",e) )
writelog(time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))+' Except:'+str_http)
df1.to_csv('record.csv',sep=',', encoding='utf-8')
writelog(traceback.format_exc())
writelog(time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))+' End:'+str_http)
time.sleep(20)
##关闭数据链接
cnx.close()
跑跑看看是不是程序一切运行正常。
