爬取抽屉热搜榜文章

import os
import requests
from bs4 import BeautifulSoup
# 登陆, 模仿用户浏览器
r1 = requests.get(
    # 要爬取的网页
    url='https://dig.chouti.com/',
    # 浏览器的信息
    headers={
        'user-agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
    }
)
# 编码
# r1.encoding = 'gbk'
# 获取第一条cookie
r1_cookie_dict = r1.cookies.get_dict()

# 登陆信息，需要的时候再加上
# r2 = requests.post(
#     url='https://dig.chouti.com/login',
#     # 加入浏览器信息
#     headers={
#         'user-agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
#     },
#     # 填写账号密码
#     data={
#         'phone':'861352039****',
#         'password':'******',
#         'oneMonth':1
#     },
#     # 抽屉网还要用第一次那个cookies值
#     cookies=r1_cookie_dict
# )

# 去响应体中j解析我们想要的数据
soup = BeautifulSoup(r1.text, 'html.parser')
# 按照规则找名字： div标签且id=content-list 找到匹配的第一个
container = soup.find(name='div', attrs={'id':'content-list'})
# 去container找到所有的class=item的div标签
div_list = container.find_all(name='div',attrs={'class':'item'})
# 循环所有的标签
for tag in div_list:
    # 找到class='part2'的所有div标签
    articles = tag.find(name='div', attrs={'class': 'part2'})
    # 如果为空就跳过
    if not articles:
        continue
    # 找到class='summary'的所有span标签
    summay = tag.find(name='span', attrs={'class': 'summary'})
    if not summay:
        continue
    # 找到class='show-content color-chag'的所有a标签
    articles_addr = tag.find(name='a', attrs='show-content color-chag')
    if not articles_addr:
        continue
    print('标题：',articles['share-title'])
    print('简介：',summay.text)
    print('文章地址：', articles_addr['href'])
    print('图片地址：', articles['share-pic'])
    print('------------------------------------------------')

    # 下载图片
    # 找到图片地址https://img.jandan.net/news/2018/07/D5BA3729EF7F435B8BD5BC7DC8DCDA8C_W550H698.jpg
    img_addr = articles['share-pic']
    # 切割出文件名D5BA3729EF7F435B8BD5BC7DC8DCDA8C_W550H698.jpg
    file_name = img_addr.rsplit('/', maxsplit=1)[1]  
    # 创建一个文件夹来存要下载的图片
    root = "F:/简单项目在这里/pacongfirst/img/"
    # join连接 拼接一个完整文件名
    path_filename = os.path.join('root', file_name)
    # 或 path_filename = root + file_name
    try:
        # 如果文件夹不存在就创建
        if not os.path.exists('root'):
            os.mkdir('root')
        # 如果文件不存在就下载
        if not os.path.exists(path_filename):
            # 拿到图片
            r3 = requests.get(
                url=img_addr
            )
            with open(path_filename, 'wb') as f:
                f.write(r3.content)
        else:
            print('文件已存在')
    except:
        print('爬取失败')