【爬虫】基本库的使用[requests]

1

import requests

'''
    实例引入
'''
r = requests.get("https://www.cnblogs.com/RickSchanze/")
print(type(r))
print(r.status_code)
print(type(r.text))
print(r.text[:100])
print(r.cookies)

'''
    GET请求
'''
r = requests.get("https://www.httpbin.org/get")
print(r.text)
# 利用params参数传递信息
data = {
    'name':'germey',
    'age':25
}
r = requests.get("https://www.httpbin.org/get", params=data)
print(r.text)

# 直接调用json方法
r = requests.get("https://www.httpbin.org/get")
print(r.text)
print(r.json())
print(type(r.json()))

2

# 抓取网页
import requests, re
r = requests.get("https://ssr1.scrape.center/")
pattern = re.compile('<h2.*?>(.*?)</h2>', re.S)
titles = re.findall(pattern, r.text)
print(titles)

# 抓取二进制数据，可解析为视频音乐等
import requests

r = requests.get("https://scrape.center/favicon.ico")
print(r.text)
print(r.content)

# 抓取二进制数据，以特定格式保存
import requests
r = requests.get("https://scrape.center/favicon.ico")
with open('favicon.ico', 'wb') as f:
    f.write(r.content)

# 设置请求头
import requests
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.60 Safari/537.36 Edg/100.0.1185.29',
#     'Host': 'www.httpbin.org'
}
r = requests.get("https://ssr1.scrape.center/", headers=headers)
print(r.text)

3

'''
    POST请求
'''
import requests
data = {
    "name":"germey",
    "age":25
}
r = requests.post("https://www.httpbin.org/post", data=data)
print(r.text)

'''
    响应
'''
import requests

r = requests.get('https://ssr1.scrape.center/')
print(type(r.status_code), r.status_code)
print(type(r.headers), r.headers)
print(type(r.cookies), r.cookies)
print(type(r.url), r.url)
print(type(r.history), r.history)

# requests.codes对象
import requests

r = requests.get("https://ssr1.scrape.center/")
exit() if not r.status_code == requests.codes.ok else print('Request Successfully')

'''
    高级用法
'''
# 文件上传
import requests
file = {
    "file":open('icon.png', 'rb')
}
r = requests.post('https://www.httpbin.org/post', files=file)
print(r.text)

4

# cookie设置
import requests

r = requests.get("https://www.cnblogs.com/RickSchanze/")
print(r.cookies)
for key, value in r.cookies.items():
    print(key + "=" + value)

# session维持
requests.get("https://www.httpbin.org/cookies/set/number/123456789")
r = requests.get("https://www.httpbin.org/cookies")
print(r.text)       # 没有获得设置的cookie

s = requests.session()
s.get("https://www.httpbin.org/cookies/set/number/123456789")
r = s.get("https://www.httpbin.org/cookies")
print(r.text)       # 成功获得设置的cookie

# SSL证书验证
import requests

response = requests.get("https://ssr2.scrape.center/")
print(response.text)        # SSLError

response = requests.get("https://ssr2.scrape.center/", verify=False)
print(response.status_code)

# 屏蔽警告
from requests.packages import urllib3
urllib3.disable_warnings()
response = requests.get("https://ssr2.scrape.center/", verify=False)
print(response.status_code)

# 输出警告到日志
import logging
logging.captureWarnings(True)
response = requests.get("https://ssr2.scrape.center/", verify=False)
print(response.status_code)

# 超时设置
import requests
r = requests.get("https://httpbin.org/get", timeout=1)  # timeout包括连接和读取的时间
print(r.status_code)
r = requests.get("https://httpbin.org/get", timeout=(5,30)) # 分别设置连接和读取时间
print(r.status_code)

5

# 身份认证
import requests
from requests.auth import HTTPBasicAuth

r = requests.get("https://ssr3.scrape.center/", auth=('admin', 'admin'))
print(r.status_code)

# prepared request
from requests import Session, Request
url = "https://www.httpbin.org/post"
data = {
    "name":"germey"
}
headers = {
    "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36 Edg/100.0.1185.36"
}
s = Session()
req = Request('POST', url, data=data, headers=headers)
prepped = s.prepare_request(req)
r = s.send(prepped)
print(r.text)