爬虫作业

（2）请用requests库的get()函数访问google网站２０次，打印返回状态，text()内容，计算text()属性和content属性所返回网页内容的长度。

import requests

url = 'http://www.google.cn/'

#请求头
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3741.400 QQBrowser/10.5.3863.400'}

def Access(url,headers = headers):
    try:
        r = requests.get(url, timeout = 30, headers = headers)
        r.raise_for_status()
        status = r.status_code
        print('响应状态码为:{}------请求访问成功'.format(status))
        return r
    except:
        print('请求访问失败')
for i in range(1,21):
    print('第{}次'.format(i))
    r = Access(url)

text = r.text
content = r.content
print('text内容:\n{}'.format(text))
print('text属性返回的网页内容长度:{}'.format(len(text)))
print('contents属性返回的网页内容长度:{}'.format(len(content)))

 

（3）这是一个简单的html页面,请保持为字符串，完成后面的计算要求。

import requests
from bs4 import BeautifulSoup
soup = BeautifulSoup("<!DOCTYPE html><html><head><meta charset=‘utf-8‘>\
                   <title菜鸟教程(rounoob.com)</title></head><body>\
                   <h1>我的第一标题</h1>\
                   <p id='first'>我的第一个段落。</p></body>\
                     <table border=‘1‘><tr><td>row 1,cell 1\
                   </td><td>row 1,cell 2</td></tr><tr><td>row 2,cell 1\
                   </td><td>row 2,cell 2</td></tr</table></html>")
 
print(soup.head,"39")                 #获取并打印head标签的内容和学号后两位
print(soup.body)                      #获取并打印body的内容
print(soup.find_all(id="first"))      #获取并打印id为first的文本
print(soup.h1.string,soup.p.string)   #获取并打印html页面中的中文字符



 

（3）爬中国大学排名网站内容

import requests
from bs4 import BeautifulSoup
allUniv = []
def getHTMLText(url):
    try:
        r = requests.get(url, timeout=30)
        r.raise_for_status()
        r.encoding = 'utf-8'
        return r.text
    except:
        return ""
def fillUnivList(soup):
    data = soup.find_all('tr')
    for tr in data:
        ltd = tr.find_all('td')
        if len(ltd)==0:
            continue
        singleUniv = []
        for td in ltd:
            singleUniv.append(td.string)
        allUniv.append(singleUniv)
def printUnivList(num):
    print("{1:^2}{2:{0}^10}{3:{0}^6}{4:{0}^4}{5:{0}^10}".format(chr(12288),"排名","学校名称","省市","总分","年费"))
    for i in range(num):
        u=allUniv[i]
        print("{1:^4}{2:{0}^10}{3:{0}^5}{4:{0}^8.1f}{5:{0}^11}".format(chr(12288),u[0],u[1],u[2],eval(u[3]),u[11]))
def main():
    url = 'http://www.zuihaodaxue.com/zuihaodaxuepaiming2017.html'
    html = getHTMLText(url)
    soup = BeautifulSoup(html, "html.parser")
    fillUnivList(soup)
    printUnivList(10)
main()

 

posted @ 2020-12-14 00:10  chengyi92  阅读(64)  评论(0)    收藏  举报