bs4:遍历文档树，搜索文档书，css选择器
爬取汽车之家新闻：
地址:https://www.autohome.com.cn/news/
目的：爬取文章的标题，图片，简介。
在这之前，学习以下bs4：https://www.cnblogs.com/liuqingzheng/p/16005875.html
代码：
import requests
from bs4 import BeautifulSoup
res = requests.get('https://www.autohome.com.cn/news/1/#liststart')
"""
第一个参数，要解析的内容，第二个餐宿，使用的解析器,html.parser bs4内置的解析器
解析器好友：lxml --》 pip3 install lxml
"""
soup = BeautifulSoup(res.text, 'html.parser')
"""
find_all 找所有,从获取到的文本中找到所有的类名为article的类
参数name:代指标签名
"""
# 从text找找所有，并且标签名为ul,类名为article
ul_list = soup.find_all(name='ul', class_="article")
# print(ul_list)
# 4 总共4个
# print(len(ul_list))
for ul in ul_list:
li_list = ul.find_all(name='li')
print(len(li_list))
for li in li_list:
# find 拿第一个，从li标签中拿第一个h3标签
h3 = li.find(name='h3')
if h3:
# 此时标签有值，可以获取到h3标签的文本
# 获取标签文本内容，标签对象，
title = h3.text
# 简介，从li标签中获取
desc = li.find(name='p').text
img_url = li.find(name='img')['src']
if not img_url.startswith('http'):
img_url = 'https:' + img_url
# print(img_url)
url = 'https:' + li.find(name='a')['href']
print(url)
print('''
新闻标题：%s
新闻摘要：%s
新闻图片：%s
新闻地址：%s
''' % (title, desc, img_url, url))
爬取不同页数内容：
# pip3 install beautifulsoup4
import requests
from bs4 import BeautifulSoup
for i in range(1,100):
res=requests.get('https://www.autohome.com.cn/news/%s/#liststart'%i)
# print(res.text)
# 第一个参数，要解析的内容，第二参数：使用的解析器 html.parser bs4内置的解析器 lxml
soup=BeautifulSoup(res.text,'html.parser')
# pip3 install lxml
# soup=BeautifulSoup(res.text,'lxml')
# find_all找所有
ul_list=soup.find_all(name='ul',class_='article')
# ul_list=soup.find_all(name='ul')
# print(len(ul_list))
for ul in ul_list:
li_list=ul.find_all(name='li')
for li in li_list:
h3=li.find(name='h3')
if h3:
title=h3.text # 获取标签的文本内容，标签对象.text
# print(title)
desc=li.find(name='p').text
# print(desc)
img_url=li.find(name='img')['src']
if not img_url.startswith('http'):
img_url='https:'+img_url
# print(img_url)
url='https:'+li.find(name='a')['href']
print(url)
print('''
新闻标题：%s
新闻摘要：%s
新闻图片：%s
新闻地址：%s
'''%(title,desc,img_url,url))
bs4遍历文档树
使用requests发送get请求之后返回html页面，可以看到，html页面没有闭合，那么bs4的基本使用来了。
<html><head><title>The Dormouse's story</title></head>
<body>
<p class="title"><b>The Dormouse's story <span>lqz</span></b><span>egon</span></p>
<p class="story">Once upon a time there were three little sisters; and their names were
<a href="http://example.com/elsie" class="sister" id="link1">Elsie</a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>
<p class="story">...</p>
bs4的基本使用，容错处理。
容错处理,文档的容错能力指的是在html代码不完整的情况下,使用该模块可以识别该错误。使用BeautifulSoup解析上述代码,能够得到一个 BeautifulSoup 的对象,并能按照标准的缩进格式的结构输出
from bs4 import BeautifulSoup
html_doc = """
<html><head><title>The Dormouse's story</title></head>
<body>
<p class="title"><b>The Dormouse's story <span>lqz</span></b><span>egon</span></p>
<p class="story">Once upon a time there were three little sisters; and their names were
<a href="http://example.com/elsie" class="sister" id="link1">Elsie</a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>
<p class="story">...</p>
"""
soup=BeautifulSoup(html_doc,'lxml') #具有容错功能
res=soup.prettify() #处理好缩进，结构化显示
print(res)
遍历文档树
# 遍历文档树：即直接通过标签名字选择，特点是选择速度快，但如果存在多个相同的标签则只返回第一个
#1、用法
#2、获取标签的名称
#3、获取标签的属性
#4、获取标签的内容
#5、嵌套选择
#6、子节点、子孙节点
#7、父节点、祖先节点
#8、兄弟节点
#遍历文档树：即直接通过标签名字选择，特点是选择速度快，但如果存在多个相同的标签则只返回第一个
html_doc = """
<html><head><title>The Dormouse's story</title></head>
<body>
<p id="my p" class="title"><b id="bbb" class="boldest">The Dormouse's story</b></p>
<p class="story">Once upon a time there were three little sisters; and their names were
<a href="http://example.com/elsie" class="sister" id="link1">Elsie</a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>
<p class="story">...</p>
"""
#1、用法
from bs4 import BeautifulSoup
# res=soup.prettify() # 美化
# print(res)
#1、用法
# html=soup.html
# title=soup.html.head.title
# title=soup.title
# print(title)
#2、获取标签的名称 ---> 标签对象.name
# a=soup.body.a
# a=soup.a.name
# print(a)
# print(soup.body.name)
#3、获取标签的属性 ---->标签对象['标签名']
# href=soup.body.a['href']
# {'href': 'http://example.com/elsie', 'class': ['sister'], 'id': 'link1'}
# attrs=soup.body.a.attrs # 所有属性，---》字典
# 字典取值方式
# href=soup.body.a.attrs['href']
# print(attrs['class'])
# c=soup.p.attrs['class']
# print(c)
#4、获取标签的内容
# res=soup.b.text # 拿到当前标签子子孙所有的text
# res=soup.p.text
# res=soup.p.string # 当前标签有且只有一个文本内容才能拿出来
# res=soup.b.string # 当前标签有且只有一个文本内容才能拿出来
# res=soup.p.strings # 把子子孙放到生成器中
#
# print(list(res))
#5、嵌套选择
# res=soup.html.body.p
# print(type(res)) # bs4.element.Tag
from bs4.element import Tag
####了解
#6、子节点、子孙节点
# print(soup.p.contents) #p下所有子节点，放到列表中
# print(soup.p.children) #得到一个迭代器,包含p下所有子节点
# for i,child in enumerate(soup.p.children):
# print(i,child)
# print(soup.p.descendants) #获取子孙节点,p下所有的标签都会选择出来
# for i,child in enumerate(soup.p.descendants):
# print(i,child)
#7、父节点、祖先节点
# print(soup.a.parent) #获取a标签的父节点
# print(soup.body.parent)
# print(soup.a.parents) #找到a标签所有的祖先节点，父亲的父亲，父亲的父亲的父亲...
# print(list(soup.a.parents))
# print(len(list(soup.a.parents)))
#8、兄弟节点
# print(soup.a.next_sibling) #下一个兄弟
# print(soup.a.previous_sibling) #上一个兄弟
#
# print(list(soup.a.next_siblings)) #下面的兄弟们=>生成器对象
# print(list(soup.a.previous_siblings)) #上面的兄弟们=>生成器对象
bs4搜索文档树
from bs4 import BeautifulSoup
html_doc = """
<html><head><title>The Dormouse's story</title></head>
<body id='body'>
<p class="title"><b>The Dormouse's story <span>lqz</span></b><span>egon</span></p>
<p class="story">Once upon a time there were three little sisters; and their names were
<a href="http://example.com/elsie" class="sister" id="link1">Elsie</a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>
<p class="story">...</p>
"""
soup=BeautifulSoup(html_doc,'html.parser')
# 搜索文档树 find find_all
# 五种过滤器: 字符串、正则表达式、列表、True、方法
##### 字符串
# res=soup.find(name='body')
# res=soup.find(name='p',class_='story')
# 查找id为link2的标签
# res=soup.find(id='link2',name='a',class_='sister',href='http://example.com/lacie')
# res=soup.find(href='http://example.com/lacie')
# print(res)
# res=soup.find(attrs={'class':['sister']})
# print(res)
#### 正则表达式
import re
# res=soup.find_all(name=re.compile('^b')) #找出b开头的标签，结果有body和b标签
# res=soup.find(name=re.compile('^b'))
# res=soup.find_all(class_=re.compile('^s'))
# res=soup.find_all(href=re.compile('^http'))
# res=soup.find_all(id=re.compile('^l'))
# print(res)
####列表、
# res=soup.find_all(name=['body','b'])
# res=soup.find_all(id=['link1','link2'])
# res=soup.find_all(attrs={'id':['link1','link2']})
#
# print(res)
# True、
# links=soup.find_all(href=True)
# print(links)
# res=soup.find_all(name=True)
# res=soup.find_all(id=True)
# print(res)
#方法
# def has_class_but_no_id(tag):
# return tag.has_attr('class') and not tag.has_attr('id')
#
# print(len(soup.find_all(name=has_class_but_no_id)))
# 拿出当前页面所有图片
soup.find_all(name='img',href=True)
## 建议 遍历文档树和搜索文档树混用
# soup.body.div.find
### 其他参数 find，find_all
#limit
# soup.find()
# res=soup.find_all(name='a',href=True,limit=2) # 限制获取的条数
# print(res)
# recursive 是否递归查找
# res=soup.find_all(name='a',recursive=False)
# res=soup.find_all(name='html',recursive=False)
# print(res)
css选择器
### css，xpath选择器是通用的---》基本所有的解析库(bs4,lxml,pyquery,selenium的解析库)--->都支持css选择器-->css在前端通用
from bs4 import BeautifulSoup
html_doc = """
<html><head><title>The Dormouse's story</title></head>
<body>
<p class="title" id="id_p">lqz<b>The Dormouse's story</b></p>
<p class="story">Once upon a time there were three little sisters; and their names were
<a href="http://example.com/elsie" class="sister" id="link1">Elsie</a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>
<p class="story">...</p>
"""
soup = BeautifulSoup(html_doc, 'lxml')
# soup.select() # 找所有
# soup.select_one() # 找一个
'''
div 找div标签
div>a 找div下的紧邻的a
div a 找div下的子子孙孙的a
.sister 找类名为sister的标签
#id_p 找id为id_p的标签
'''
# res=soup.p.select('.sister') # 使用css选择器
# res=soup.p.select('#link1') # 使用css选择器
# res=soup.select('body>p') # 使用css选择器 body的子标签p
res=soup.select('body p') # 使用css选择器 body的子子孙孙标签p
print(len(res))
# res=soup.select('#id_p')
# res=soup.select('.sister')
# res=soup.select_one('.story>a').attrs.get('href')
# print(res)
# 终极大招
import requests
response=requests.get('https://www.runoob.com/cssref/css-selectors.html')
soup=BeautifulSoup(response.text,'lxml')
res=soup.select_one('#content > table > tbody > tr:nth-child(2) > td:nth-child(3)').text
print(res)
# 只要页面中有的通过bs4都能解析出来
