Python爬虫基础

常用对象

一、爬虫过程

import re
import requests

# 发送请求,查看响应的各个属性
url = "http://www.buu.edu.cn"
resp = requests.get(url)     

print(resp.status_code)    # 响应码 
print(resp.url)
print(resp.headers)
print(resp.request)
print(resp.content)
print(resp.encoding)   # ISO-8859-1
print(resp.apparent_encoding)   # utf-8

resp.encoding = resp.apparent_encoding

def getHtml(url):
    try:
        resp = requests.get(url)
        resp.raise_for_status()      # 网络连接异常
        resp.encoding = resp.apparent_encoding   # 设置编码方式
    except:
        print("error")
    return resp.text     # 返回响应的内容

html = getHtml(url)
print(html)

二、封装请求

import requests

# 模拟浏览器和定制请求
url = "http://www.douban.com"
resp = requests.get(url)

# 查看属性
print(resp.text)
print(resp.request.headers)

# 封装请求
header = {
    "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, likeGecko) Chrome/92.0.4515.131 Safari/537.36"
}

# 发送请求
resp = requests.get(url,headers=header)
print(resp.text)   # 网页源代码

 

三、BS解析

from bs4 import BeautifulSoup # 解析网页
import re

# 创建一个BeautifulSoup对象,参数: 打开的要解析的文件
soup = BeautifulSoup(open("index.html",encoding='utf-8'))

# 字符串
for tag in soup.find_all("p"):
    print(tag.name)

# 正则表达式 ,compile编写
for tag in soup.find_all(re.compile("^b")):   # 以b开头
    print(tag.name)

for tag in soup.find_all(re.compile("e")):     # 含有e
    print(tag.name)

# 列表,["",""]
ls=["a","p"]
for tag in soup.find_all(ls):
    print(tag)                    # 显示的是内容+标签

# True 全部的标签
for tag in soup.find_all(True):
    print(tag.name) 


# find_all()方法
# 标签名查找
res = soup.find_all("p") 
print(res)

# 标签属性
res = soup.find_all(border="1")
print(res)

res = soup.find_all(attrs={"border":"1"})    # 多个参数时,常用{"":""}
print(res)

# CSS选择器
res = soup.find_all(class_="me")
print(res)

# 标签内容
res = soup.find_all(text="大数据导论")
print(res)

# 组合查找
res = soup.find_all("td",border="1")   # 指明
print(res)

res = soup.find_all(text=re.compile("数据"))   # 赋值内容 
print(res)

res = soup.find_all(href=re.compile("^http:"))  # 查找链接
print(res)

 

四、选中元素

import re
from bs4 import BeautifulSoup

soup = BeautifulSoup(open("index.html",encoding="utf-8"))

'''
select() 方法

'''
# 单个标签名查找
res = soup.select("a")
print(res)

# 多个标签层层查找
res = soup.select("p a")
print(res)

res = soup.select("body table th")
print(res)

# 按属性查找
res = soup.select("a[href]")
print(res)


# 增加上属性值
res = soup.select("a[href='http://www.buu.edu.cn']")
print(res)

res = soup.select('a[href*="www"]')  # 含有www
print(res)

res = soup.select("a[href='http://']")
print(res)

res = soup.select("a[href$='.cn']")
print(res)


# CSS选择器
# id选择器
res = soup.select("#t1")
print(res)

# 类名选择器
res = soup.select(".me")
print(res)

# 同时用多种CSS选择器查询元素
res = soup.select("#t1,.me")
print(res)

 

五、获取内容

from bs4 import BeautifulSoup

soup = BeautifulSoup(open("index.html",encoding="utf-8"))

print(soup.prettify)   # 页面内容

print(soup.a)
print(soup.table)
print(soup.table.name)

tattrs = soup.table.attrs    # 属性

print(tattrs)
print(tattrs["border"]) # 取出其中的一个属性

print(type(soup.a))    # 查看类型

print(soup.a.string)    # 查看超链接的内容

print(soup.contents)    # 源代码

print(soup.table.tr)

 

六、遍历树

from bs4 import BeautifulSoup

soup = BeautifulSoup(open("index.html",encoding="utf-8"))

# 文档树的遍历
print(soup.table.parent) # 源代码

tags = soup.table.parents  # 所有的父元素
for tag in tags:
    print(tag.name)

# 
print(soup.table.children) # 子元素


# 
print(soup.table.descendants) # 后代