python requests bs4入门（一）-获取TOP100榜电影名字和主演写入数据库

​Requests

获取猫眼TOP100榜电影名字和主演

​

import time
import requests
from model import *
from bs4 import BeautifulSoup
headers = {"Content-Type": "text/html; charset=utf-8",
           "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36"}

url = f"https://www.maoyan.com/board/4?timeStamp=1637291300330&channelId=40011&index=2&signKey=c286a92c2bb667036254185fde905f09&sVersion=1"
r = requests.get(url, timeout=3, headers=headers)
print(url)
soup = BeautifulSoup(r.text, "html5lib")
bb = soup.find('ul', class_="list-pager").text.replace("\n", "").split(" ")
page = [i for i in bb if i != '']
print(page[-2])
for i in range(0,int(page[-2])):
    p=i*10
    print(i)
    url = f"https://www.maoyan.com/board/4?timeStamp=1637053092611&sVersion=1&index=7&signKey=52df1051b8c1478e914905882e09e10a&channelId=40011&requestCode=c30fbaba9d9f7b73a53f83fe71ac0ec13ztmp&offset={p}"
    r = requests.get(url, timeout=3, headers=headers)
    print(url)
    time.sleep(3)
    print(r.status_code)
    soup = BeautifulSoup(r.text, "html5lib")
    # 获取某标签的属性值
    soup1 = BeautifulSoup(r.text, "html5lib")
    aa = soup.find_all('div', class_="board-item-content")
    for a in aa:
        bb = []
        datalist={}
        dd = a.find('p', class_="name")
        ff = a.find('p', class_="star")
        ss = a.find('p', class_="releasetime")

        movie_name=dd.getText().replace("\n","")
        art_name=ff.getText().replace("\n","").replace(" ","").replace("主演：","")
        movie_time=ss.getText().replace("\n", "").replace(" ", "").replace("上映时间：","")
        print(movie_name)
        print(art_name)
        print(movie_time)
        bb.append(dict(movie_name=movie_name, art_name=art_name, movie_time=movie_time, remark='猫眼'))
        for l in bb:
            find_data = SQLsession.query(Infos).filter_by(movie_name=l['movie_name'], remark='猫眼').first()
            if not find_data:
                SQLsession.add(Infos(**l))
        SQLsession.commit()

from sqlalchemy import *
import pymysql
from sqlalchemy.orm import sessionmaker
from sqlalchemy.ext.declarative import declarative_base
from datetime import datetime

database = 'mysql+pymysql://root:1234@localhost/test_ten?charset=utf8mb4'

Base = declarative_base()
# 创建数据库连接对象
engine = create_engine(database)
DBSession = sessionmaker(bind=engine)
SQLsession = DBSession()


# ORM
class Infos(Base):
    __tablename__ = 'test_table1'
    id = Column(Integer(), primary_key=True)
    code = Column(String(255))
    movie_name = Column(String(255))
    art_name = Column(String(255))
    movie_time=Column(String(255))
    status = Column(Integer(), default=1)
    remark = Column(Text)
    created = Column(DateTime, default=datetime.now())
    updated = Column(DateTime, default=datetime.now(), onupdate=datetime.now())


Base.metadata.create_all(engine)

​

 ​

如果出现以下情况点进去滑动验证重新运行就可以正常获取数据

​

豆瓣电影

​

import requests
from bs4 import BeautifulSoup
url='https://movie.douban.com/chart'
headers={"Content-Type":"text/html; charset=utf-8",
         "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36"}
print(url)
r=requests.get(url, timeout=3, headers=headers)
soup=BeautifulSoup(r.text,"html5lib")
name=soup.find_all('div',class_='pl2')
for i in name:
    aa=i.getText().replace("\n","").replace("/","")
    print(aa)

​

Requests-HTML

# 使用Requests-HTML模拟Ajax请求来获取网页数据
from requests_html import HTMLSession
url='https://movie.douban.com/chart'
headers={"Content-Type":"text/html; charset=utf-8",
         "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36"}
print(url)
# Requests-HTML的Ajax加载功能，Requests-HTML只能使用Requests的Session模式
session=HTMLSession()
r=session.get(url)
# Chromium浏览器加载网页
# Ajax加载功能由render（）方法实现
r.html.render()
name=r.html.find('div.pl2')
for g in name:
    print(g.text)

​

json解析

参考：python的requests爬取Json数据，从Json数据中提取标题和图片（腾讯视频的某综艺节目）_Mr.Pan_学狂-CSDN博客

​

 

​

 

​

 ​

# 使用Requests-HTML模拟Ajax请求来获取网页数据
from requests_html import HTMLSession
import json
url='https://movie.douban.com/tag/#/'
headers={"Content-Type":"text/html; charset=utf-8",
         "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36"}
print(url)
# Requests-HTML的Ajax加载功能，Requests-HTML只能使用Requests的Session模式
session=HTMLSession()
r=session.get(url)
# Chromium浏览器加载网页
# Ajax加载功能由render（）方法实现
r.html.render()
# name=r.html.find('span.title')
# for g in name:
#     print(g.text)
for d in range(0,5):
    p = d * 20
    url1=f"https://movie.douban.com/j/new_search_subjects?sort=U&range=0,10&tags=&start={p}"
    r2 = session.get(url1)
    print(r2.text)
    json_data = json.loads(r2.text)  # 加载内容
    Data_ls = json_data['data'] # 取出data对应的值，是list列表类型
    title_dt = {}  # 定义一个空字典，用于存储标题和对应的演员
    for D in Data_ls:
        try:  # 加入异常处理机制，使不满足条件的时候跳过，而不报错。
            if type(D) is dict:
                title_dt[D['title']] = D['casts']
        except Exception:  # 出现异常的时候，跳过
            continue
    print(title_dt)#打印字典

​

​