python爬虫

Requests

安装和导入

#安装
pip install requests

#导入
import requests

高级GET请求

# 随机获取User-Agent 模块
from fake_useragent import UserAgent
# requests模块
import requests

# User-Agent 实例化一个对象
ua = UserAgent()

# 重构请求头
heads = {
    "User-Agent": ua.ie,
    # "Cookie": "......."
}

# 要爬的网址
url = 'http://httpbin.org/get'

# requests GET请求
requ = requests.get(url, headers=heads)

data = requ.text #取文本数据

print(data)

模块随机获取 User-Agent

模块安装

pip install fake-useragent

演示代码

from fake_useragent import UserAgent
#实例化一个对象
ua=UserAgent()
#随机获取一个ie浏览器ua
print(ua.ie)
print(ua.ie)
#随机获取一个火狐浏览器ua
print(ua.firefox)
print(ua.firefox)

URL编码

from urllib import parse



#编码
#简单的
st = "中标通告"
st = parse.quote(st, encoding="gbk")
print(st)

#数据对象的
data = {
    "title": "短袖",
    "relItemId": 123456,
    "categoryId": 12
}
st = parse.quote(str(data), encoding="gbk")
print(st)


#解码
st = parse.unquote("%D6%D0%B1%EA%CD%A8%B8%E6", encoding="gbk")
print(st)