urllib3的使用

urllib3 的使用

• 安装： pip install urllib3
  urllib3 是一个基于Python3的功能强大，友好的http客户端

import urllib3
# 实例化 PoolManager 对象构造请求。这个对象处理了连接池和线程安全的所有细节，不用我们自行处理
http = urllib3.PoolManager()
# 发送请求
response = http.request('GET', 'http://httpbin.org/get', fields={'user': 'long', 'love': 'hello'})
print(response.status)   # 状态码
print(response.data.decode('utf-8'))   # 数据
print(response.headers)  # 响应头

200
{
  "args": {
    "love": "hello", 
    "user": "long"
  }, 
  "headers": {
    "Accept-Encoding": "identity", 
    "Host": "httpbin.org", 
    "X-Amzn-Trace-Id": "Root=1-5e3541ae-002a2868da392ad037911930"
  }, 
  "origin": "180.137.6.157", 
  "url": "http://httpbin.org/get?user=long&love=hello"
}

HTTPHeaderDict({'Date': 'Sat, 01 Feb 2020 09:15:26 GMT', 'Content-Type': 'application/json', 'Content-Length': '299', 'Connection': 'keep-alive', 'Server': 'gunicorn/19.9.0', 'Access-Control-Allow-Origin': '*', 'Access-Control-Allow-Credentials': 'true'})
http://httpbin.org/get

def post_test():
    url = 'http://httpbin.org/post'
    http = urllib3.PoolManager()
    response = http.request('POST', url, fields={'user': 'long', 'pwd': 123456})
    print(response.data.decode())
post_test()

{
  "args": {}, 
  "data": "", 
  "files": {}, 
  "form": {
    "pwd": "123456", 
    "user": "long"
  }, 
  "headers": {
    "Accept-Encoding": "identity", 
    "Content-Length": "217", 
    "Content-Type": "multipart/form-data; boundary=3d17043d15a2ec51d2baf65ca6cec7c2", 
    "Host": "httpbin.org", 
    "X-Amzn-Trace-Id": "Root=1-5e353979-f2194286f61c9ceae23fe342"
  }, 
  "json": null, 
  "origin": "180.137.6.157", 
  "url": "http://httpbin.org/post"
}

​

response.stream()

response: 响应返回的数据都是字节类型，对于大量的数据我们通过 stream 来处理更好

def big_data():
    http = urllib3.PoolManager()
    response = http.request('GET', 'http://httpbin.org/bytes/1024', preload_content=False)
    for chunk in response.stream(32):
        print(chunk)
# big_data()

也可以当做一个文件对象来处理

# 也可以当做一个文件对象来处理
def file_test():
    http = urllib3.PoolManager()
    response = http.request('GET', 'http://httpbin.org/bytes/1024', preload_content=False)
    for line in response:
        print(line)
# file_test()

使用代理

可以使用urllib3.ProxyManager 进行代理操作

proxy = urllib3.ProxyManager('http://106.85.139.161:9999')
response = proxy.request('GET', 'http://httpbin.org/ip')
# print(response.status)
# print(response.data.decode('utf-8'))

添加请求头

发送请求时传递字典形式的headers参数去指定请求头

headers = {
    'user-agnet': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.80 Safari/537.36'
}
http = urllib3.PoolManager()
response = http.request('GET', 'http://httpbin.org/headers', headers=headers)
print(response.data.decode())

{
  "headers": {
    "Accept-Encoding": "identity", 
    "Host": "httpbin.org", 
    "User-Agnet": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.80 Safari/537.36", 
    "X-Amzn-Trace-Id": "Root=1-5e353fcc-7bd4f0203529ac98d781b4e8"
  }
}