Python爬虫 Day 3.1 

二.BeautifulSoup4  (BS4)
       1.什么是BeautifulSoup4
           BS4是一个解析库，可以通过某种解析器来帮我们提取我们想要的数据

       2.为什么要使用BS4
           因为它可以通过简洁的语法快速提取用户想要的数据内容

       3.解析器的分类
           lxml HTML解析器（首选）
           Python标准库（其次）
           lxml XML解析器
           html5lib

       4.安装与使用
           遍历文档树

          搜索文档树

补充知识点：
生成器：（把值放进生成器中）
    def f():
        # return 1
        yield 1
        yield 2
        yield 3
    g = f()
    print(g)
    
    for line in g:
        print(line)

'''
安装解析器
pip3 install lxml
安装解析库
pip3 install bs4
'''
html_doc = """
<html><head><title>The Dormouse's story</title></head>
<body>
<p class="title"><b>The Dormouse's story</b></p>

<p class="story">Once upon a time there were three little sisters; and their names were
<a href="http://example.com/elsie" class="sister" id="link1">Elsie</a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>

<p class="story">...</p>
"""

from bs4 import BeautifulSoup
# python自带的解析库
# soup = BeautifulSoup(html_doc,'html.parser')

# 调用bs4得到一个soup对象
soup = BeautifulSoup(html_doc,'lxml')

# bs4对象
print(soup)

#bs4类型
print(type(soup))

# 美化功能
html = soup.prettify()
print(html)
html_doc = """
<html><head><title>The Dormouse's story</title></head>
<body>
<p class="sister"><b>$37</b></p>

<p class="story" id="p">Once upon a time there were three little sisters; and their names were
<a href="http://example.com/elsie" class="sister" >Elsie</a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>

<p class="story">...</p>
"""
from bs4 import BeautifulSoup

soup = BeautifulSoup(html_doc,'lxml')
# print(soup)
# print(type(soup))

# 遍历文档树
# 1、直接使用 *****
print(soup.html)
print(type(soup))
print(soup.a)
print(soup.p)

# 2、获取标签的名称
print(soup.a.name)

# 3、获取标签的属性   *****
print(soup.a.attrs)  #获取a标签中所有的属性
print(soup.a.attrs['href'])

# 4、获取标签的内容
print(soup.p.text)

# 5、嵌套选择
print(soup.html.body.p)

# 6、子节点、子孙节点
print(soup.p.children)    #返回迭代器对象
print(list(soup.p.children))

# 7、父节点、祖先节点
print(soup.b.parent)
print(soup.b.parents)

# 8、兄弟节点  （sibling: 兄弟姐妹）
print(soup.a)
# 获取下一个兄弟节点
print(soup.a.next_sibling)
# 获取下一个的所有兄弟节点,返回的是一个生成器
print(soup.a.next_siblings)
print(list(soup.a.next_siblings))

# 获取上一个兄弟节点
print(soup.a.previous_sibling)
# 获取上一个的所有兄弟节点，返回的是一个生成器
print(list(soup.a.previous_siblings))

 

'''
find :
find_all:
标签查找与属性查找:
            name 属性匹配
            name 标签名
            attrs 属性查找匹配
            text 文本匹配
    标签:
        - 字符串过滤器   字符串全局匹配


        - 正则过滤器
            re模块匹配

        - 列表过滤器
            列表内的数据匹配

        - bool过滤器
            True匹配

        - 方法过滤器
            用于一些要的属性以及不需要的属性查找。

    属性:
        - class_
        - id
'''
html_doc = """
<html><head><title>The Dormouse's story</title></head><body><p class="sister"><b>$37</b></p><p class="story" id="p">Once upon a time there were three little sisters; and their names were<a href="http://example.com/elsie" class="sister" >Elsie</a><a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>and they lived at the bottom of a well.</p><p class="story">...</p>
"""
from bs4 import BeautifulSoup
soup = BeautifulSoup(html_doc,'lxml')

# name 标签名
# attrs 属性查找匹配
# text 文本匹配
# find 与find_all搜索文档

'''
字符串过滤器
'''
# p = soup.find(name='p')
# p_s = soup.find_all(name='p')
#
# print(p)
# print(p_s)

# name + attrs
# p = soup.find(name='p',attrs={"id":"p"})
# print(p)

# name + text
# tag = soup.find(name='title',text="The Dormouse's story")
# print(tag)

# name + attrs + text
tag = soup.find(name='a',attrs={"class":"sister"},text="Elsie")
print(tag)

'''
 - 正则过滤器
   re模块匹配
'''
import re
# name
# 根据re模块匹配带有a的节点
a = soup.find(name=re.compile('a'))
print(a)

a_s = soup.find_all(name=re.compile('a'))
print(a_s)

# attrs
a= soup.find(attrs={"id":re.compile('link')})
print(a)

# - 列表过滤器
#     列表内的数据匹配
print(soup.find(name=['a','p','heml',re.compile('a')]))
print(soup.find_all(name=['a','p','heml',re.compile('a')]))

#  - bool过滤器
#     True匹配
print(soup.find(name=True,attrs={"id":True}))

# - 方法过滤器
#     用于一些要的属性以及不需要的属性查找。
def have_id_not_class(tag):
    # print(tag.name)
    if tag.name == 'p' and tag.has_attr("id") and not tag.has_attr("class"):
        return tag

# # print(soup.find_all(name=函数对象))
print(soup.find_all(name=have_id_not_class))

# 补充知识点：
# id
a = soup.find(id='link2')
print(a)

# class
p = soup.find(class_='sister')
print(p)

 

posted @ 2019-07-03 20:09  MerliahSwift  阅读(163)  评论(0)    收藏  举报