爬虫数据分析（bs4,xpath,正则表达式)

1、bs4的BeautifulSoup函数的使用：将获取的网页格式的文本解析，之后获取想要的数据

from bs4 import BeautifulSoup
import requests

headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.74 Safari/537.36'}
url = "http://sanguo.5000yan.com/"
page_text = requests.get(url ,headers = headers).content
## 传入解析器：lxml 比如解析成中文 排版
soup = BeautifulSoup(page_text,'lxml')
#从解析之后的内容取值想要的内容： 有几种方式方法 --可搜索 <soup.select()函数的使用用法> 查看
li_list = soup.select('main > div > ul > li')
fp = open('./sanguo.txt','w',encoding='utf-8') #打开一个文件 如果没有的话就创建一个
for li in li_list:
    #下面这段的意思 <li class="menu-item"><a href="https://sanguo.5000yan.com/1083.html">第一百一十九回 假投降巧计成虚话 再受禅依样画</a></li>  ------->>>>> <a href="https://sanguo.5000yan.com/965.html">第一回 宴桃园豪杰三结义 斩黄巾英雄首立功</a> ----->>>>> 第一回 宴桃园豪杰三结义 斩黄巾英雄首立功
    title = li.a.string # 取a 里面的标题
    #取a中的网址 进行深入二度爬虫
    detail_url = li.a['href']
    detail_page_text = requests.get(detail_url,headers = headers).content
    detail_soup = BeautifulSoup(detail_page_text,'lxml') #传入解析器 解析排版
    div_tag = detail_soup.find('div',class_="grap") #抓取想要的内容
    content = div_tag.text
    fp.write(title+":"+content+'\n') #写入文件里面
    print(title,'爬取成功！！！')

2、xpath的使用

import requests
from lxml import etree
import os

headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.74 Safari/537.36'}
url = 'http://pic.netbian.com/4kmeinv/'
response = requests.get(url,headers = headers)
#response.encoding=response.apparent_encoding
#response.encoding = 'utf-8'
page_text = response.text
tree = etree.HTML(page_text)  #etree.HTML()可以用来解析字符串格式的HTML文档对象，将传进去的字符串转变成_Element对象。作为_Element对象，可以方便的使用getparent()、remove()、xpath()等方法。
print(tree)
# print(page_text)
li_list = tree.xpath('//div[@class="slist"]/ul/li')#文本如下所示
'''    <div class="slist">
       <ul class="clearfix">
        <li><a href="/tupian/28468.html" target="_blank"><img src="/uploads/allimg/211219/114328-1639885408db64.jpg" alt="º£ÇÙÑÌ Í¬ÈË cosplayÃÀÅ®4K¸ßÇå±ÚÖ½" /><b>º£ÇÙÑÌ Í¬ÈË cosplayÃÀÅ®</b></a></li>
      </ul>
    </div>
'''
if not os.path.exists('./picLibs'):
    os.mkdir('./picLibs')
for li in li_list:
    img_src = 'http://pic.netbian.com/'+li.xpath('./a/img/@src')[0] #获取网址 爬虫网页
    img_name = li.xpath('./a/img/@alt')[0]+'.jpg'
    img_name = img_name.encode('iso-8859-1').decode('gbk')#转换成中文
    img_data = requests.get(url = img_src,headers = headers).content
    img_path ='picLibs/'+img_name
    #保存图片
    with open(img_path,'wb') as fp:
        fp.write(img_data)
        print(img_name,"下载成功")

3、正则表达式 --分页爬取

import requests
import re
import os

if not os.path.exists('./qiutuLibs'):
    os.mkdir('./qiutuLibs')
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4385.0 Safari/537.36'
}

url = 'https://www.qiushibaike.com/imgrank/page/%d/'
for pageNum in range(1,3):
    new_url = format(url%pageNum)
    page_text = requests.get(new_url,headers = headers).text
    ex = '<div class="thumb">.*?<img src="(.*?)" alt.*?</div>'
    img_src_list = re.findall(ex,page_text,re.S)
    print(img_src_list)
    for src in img_src_list:
        src = 'https:' + src
        img_data = requests.get(url = src,headers = headers).content
        img_name = src.split('/')[-1]
        imgPath = './qiutuLibs/'+img_name
        with open(imgPath,'wb') as fp:
            fp.write(img_data)
            print(img_name,"下载完成！！！！！")