1 爬虫通用框架
2 import requests
3
4 def get_html_text(url):
5 try:
6 r =requests.get(url,timeout=20)
7
8 r.raise_for_status()
9
10 r.encoding = r.apparent_encoding
11
12 return r.text
13
14 except:
15 return "产生异常"
16
17 if __name__ == '__main__':
18 url ="http://www.baidu.com"
19 print(get_html_text(url))
20
21
22 实例
23 import requests
24 from bs4 import BeautifulSoup
25
26 def getHTMLText(url):
27 try:
28 r = requests.get(url,timeout=20) #设置超时
29 r.raise_for_status() # 判断请求是否成功
30 r.encoding = r.apparent_encoding # 设置编码
31 return r.text # 返回获取内容
32 except: #异常处理
33 return "产生异常"
34
35 if __name__ == '__main__':
36 url = "https://book.douban.com/subject/1084336/comments/" # 需要请求的网址
37 # print(getHTMLText(url)) #调用函数
38 requests = getHTMLText(url) # 获取文本内容
39 soup = BeautifulSoup(requests,"html.parser") # 文本解析
40 reasult = soup.find_all("div",class_="comment") # 文本处理
41 for i in reasult :
42 print(i.p.text) # 循坏打印文本
