urllib 库

一、get和post请求获取数据

获取网址源码

import urllib.request
response = urllib.request.urlopen("https://www.baidu.com/")
print(response.read().decode('utf-8'))

get请求

import urllib.request
import urllib.parse
response = urllib.request.urlopen("http://httpbin.org/get")
print(response.read().decode("utf-8"))

post请求

import urllib.request
import urllib.parse
data = bytes(urllib.parse.urlencode({"hello":"world"}),encoding="utf-8")
response = urllib.request.urlopen("http://httpbin.org/post",data=data)
print(response.read().decode("utf-8"))

二、超时处理

import urllib.request
import urllib.parse
try:
response = urllib.request.urlopen("http://httpbin.org/get",timeout = 0.01)
print(response.read().decode("utf-8"))
except urllib.error.URLError as e:
print("timeout!")

三、获取状态码

import urllib.request
import urllib.parse
response = urllib.request.urlopen("http://httpbin.org/get")
print(response.status)

四、浏览器伪装

1.找到浏览器信息

2.封装信息

import urllib.request
import urllib.parse

url = "https://www.douban.com"
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36"
}
req = urllib.request.Request(url=url,headers=headers)
response = urllib.request.urlopen(req)
print(response.read().decode("utf-8"))