爬虫笔记

网址中文转译

import urllib.parse
import string
3 new_url = urllib.parse.quote(new_url, safe=string.printable)  #将包含汉字的网址转译

 利用字典拼接url（字典传参）

import urllib.parse
import string
url = "http://www.baidu.com/s?"
dic = {"wd":"百度"， “key”:"zhang"}
_url = urllib.parse.urlencode(dic)#可以将字典中：转译为=
new_url = url + _url
end_url = urllib.parse.quote(new_url, safe = "string.printable")#中文转译

 动态传入User-agent

import urllib.request
import random

def load_baidu():

    url = "http://www.baidu.com"
    user_agent_list = [
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1",
        "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
        "Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.9.168 Version/11.50"

    ]
    #每次请求的浏览器都是不一样的
    random_user_agent = random.choice(user_agent_list)

    request = urllib.request.Request(url)

    #增加对应的请求头信息(user_agent)
    request.add_header("User-Agent",random_user_agent)

    #请求数据
    response = urllib.request.urlopen(request)
    #请求头的信息
    print(request.get_header("User-agent"))

load_baidu()

 利用不同ip请求网络数据

import urllib.request

def proxy_user():
    #创建代理ip列表用于轮流访问
    proxy_list = [
        {"https": "106.75.226.36:808"},
        {"https": "61.135.217.7:80"},
        {"https": "125.70.13.77:8080"},
        {"https": "118.190.95.35:9001"}
    ]
    for proxy in proxy_list:
        print(proxy)
        #利用遍历出来的ip创建处理器
        proxy_handler = urllib.request.ProxyHandler(proxy)
        #创建opener
        opener = urllib.request.build_opener(proxy_handler)

        try:
            data = opener.open("http://www.baidu.com", timeout=1)
            #使用创建出来的opener的open函数
            haha = data.read()
            print(haha)
        except Exception as e:
            print(e)


proxy_user()