Python爬虫学习系列二：发送请求并获取内容

发送请求并获取内容

Python发送请求,常用的包有两个,urllib和requests

urllib请求

#!/usr/bin/python3
# -*- encoding: utf-8 -*-
'''
@File        :urllib请求.py
@Time        :2020/09/06 22:51:52
@Author      :He
@Software    :vsCode
'''

from urllib.request import urlopen, Request
import random

user_agents = [
    "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
    "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4238.0 Safari/537.36"
]

headers = {
    'User-Agent': random.choice(user_agents)
}

url = "http://www.baidu.com/"
request = Request(url, headers=headers)
# 发送请求
resp = urlopen(request)
# 请求状态
if resp.getcode() == 200:
    # 读取内容    decode()默认UTF-8
    info = resp.read().decode()
    # print(info)

    # 返回实际访问的url
    print(resp.geturl())

requests

#!/usr/bin/python3
# -*- encoding: utf-8 -*-
'''
@File        :requests请求.py
@Time        :2020/09/06 22:52:30
@Author      :He
@Software    :vsCode
'''

from requests import get
import random

user_agents = [
    "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
    "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4238.0 Safari/537.36"
]

headers = {
    'User-Agent': random.choice(user_agents)
}

url = "http://www.baidu.com/"
# 发送请求
resp = get(url, headers=headers)
# 打印请求状态    200代表成功
print(resp.status_code)
# 接收内容
if resp.status_code == 200:
    info = resp.text

完整代码,查看码云