爬取猫眼电影影评
主题：对即将上映的大侦探皮卡丘电影保持什么态度？
主要内容
蒂姆·古德曼（贾斯提斯·史密斯 饰） 为寻找下落不明的父亲来到莱姆市，意外与父亲的前宝可梦搭档大侦探皮卡丘（瑞恩·雷诺兹 配音）相遇，并惊讶地发现自己是唯一能听懂皮卡丘说话的人类，他们决定组队踏上揭开真相的刺激冒险之路。探案过程中他们邂逅了各式各样的宝可梦，并意外发现了一个足以毁灭整个宝可梦宇宙的惊天阴谋。
爬取对象：猫眼电影影评
爬取限制：pc端无法获取影评（移动端可以）
爬取内容：
爬取评论部分的用户ID、用户名、评论、评分、时间五项。
爬取的json数据切入口：http://m.maoyan.com/mmdb/comments/movie/346629.json?_v_=yes&offset=0&startTime=2019-05-09%2022%3A25%3A03
爬取结果存入CSV以及数据库
词频及词语显示
评论者性别分析
这部电影除去未知性别的，在已知性别的评论者男性的比例比较多，说明这部电影男性的
爱好者比较多。
评论者评分等级分析
根据上面分饼图可得满分的占了70%左右，4.5分以上占了7.4%左右，可知这部电影的
评价十分高，应该是非常好看的，值得去观看。
城市分布显示
总结
对于此次影评的分析，可以看出在即将上映的前夕，大部分影迷对于这部电影怀抱着回忆童年的心态，皮卡丘的名字被大多数人提及，证明绝大部分群体应该都观看过宠物小精灵，决大部分人对这部电影充满了期待，从城市分布可以看出观影群体主要以一二线城市为主。
全部代码
import requests
from bs4 import BeautifulSoup
from datetime import datetime
import re
import sqlite3
import pandas as pd
import time
import pandas
import random
import json
#设置合理的user-agent，爬取数据函数
def getData(url):
headers =[
{'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36','Cookie': '_lxsdk_cuid=16a8d7b1613c8-0a2b4d109e58f-b781636-144000-16a8d7b1613c8; _lx_utm=utm_source%3DBaidu%26utm_medium%3Dorganic; uuid_n_v=v1; iuuid=1BB9A320700C11E995DE7D45B75E59C6FC50A50D996543D0819E9EB2E6507E92; webp=true; ci=20%2C%E5%B9%BF%E5%B7%9E; selectci=; __mta=45946523.1557151818494.1557367174996.1557368154367.23; _lxsdk=1BB9A320700C11E995DE7D45B75E59C6FC50A50D996543D0819E9EB2E6507E92; __mta=45946523.1557151818494.1557368154367.1557368240554.24; from=canary; _lxsdk_s=16a9a2807fa-ea7-e79-c55%7C%7C199'},
{ 'User-Agent': 'Mozilla / 5.0(Linux;Android 6.0; Nexus 5 Build / MRA58N) AppleWebKit / 537.36(KHTML, like Gecko) Chrome / 73.0 .3683.103Mobile Safari / 537.36','Cookie':'_lxsdk_cuid=16a8d7b1613c8-0a2b4d109e58f-b781636-144000-16a8d7b1613c8; _lx_utm=utm_source%3DBaidu%26utm_medium%3Dorganic; uuid_n_v=v1; iuuid=1BB9A320700C11E995DE7D45B75E59C6FC50A50D996543D0819E9EB2E6507E92; webp=true; ci=20%2C%E5%B9%BF%E5%B7%9E; selectci=; __mta=45946523.1557151818494.1557367174996.1557368154367.23; _lxsdk=1BB9A320700C11E995DE7D45B75E59C6FC50A50D996543D0819E9EB2E6507E92; __mta=45946523.1557151818494.1557368154367.1557368240554.24; from=canary; _lxsdk_s=16a9a2807fa-ea7-e79-c55%7C%7C199'},
{'User-Agent': 'Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10','Cookie':'_lxsdk_cuid=16a8d7b1613c8-0a2b4d109e58f-b781636-144000-16a8d7b1613c8; _lx_utm=utm_source%3DBaidu%26utm_medium%3Dorganic; uuid_n_v=v1; iuuid=1BB9A320700C11E995DE7D45B75E59C6FC50A50D996543D0819E9EB2E6507E92; webp=true; ci=20%2C%E5%B9%BF%E5%B7%9E; selectci=; __mta=45946523.1557151818494.1557367174996.1557368154367.23; _lxsdk=1BB9A320700C11E995DE7D45B75E59C6FC50A50D996543D0819E9EB2E6507E92; __mta=45946523.1557151818494.1557368154367.1557368240554.24; from=canary; _lxsdk_s=16a9a2807fa-ea7-e79-c55%7C%7C199'}
]
# proxies = [{'https': 'https://120.83.111.194:9999','http':'http://14.20.235.120:808'},{"http": "http://119.131.90.115:9797",
# "https": "https://14.20.235.96:9797"}]
get=requests.get(url, headers=headers[random.randint(0,2)]);
get.encoding = 'utf-8'
return get
#数据处理函数
def dataProcess(data):
data = json.loads(data.text)['cmts']
allData = []
for i in data:
dataList = {}
dataList['id'] = i['id']
dataList['nickName'] = i['nickName']
dataList['cityName'] = i['cityName'] if 'cityName' in i else '' # 处理cityName不存在的情况
dataList['content'] = i['content'].replace('\n', ' ', 10) # 处理评论内容换行的情况
dataList['score'] = i['score']
dataList['startTime'] = i['startTime']
if "gender" in i:
dataList['gendar'] = i["gender"]
else:
dataList['gendar'] = i["gender"] = 0
allData.append(dataList)
return allData
allData=[]
for i in range(67):
get=getData('http://m.maoyan.com/mmdb/comments/movie/346629.json?_v_=yes&offset={}&startTime=2019-05-09%2022%3A25%3A03'.format(i*15))
allData.extend(dataProcess(get))
#处理后的数据保存为csv文件
pd.Series(allData)
newsdf=pd.DataFrame(allData)
newsdf.to_csv('news.csv',encoding='utf-8')
# #把csv文件保存到sqlite
# newsdf = pd.read_csv('news.csv')
# with sqlite3.connect('sqlitetest.sqlite') as db:
# newsdf.to_sql('data',con = db)
# 评论者性别分布可视化
def sexProcess(gender):
from pyecharts import Pie
list_num = []
list_num.append(gender.count(0)) # 未知
list_num.append(gender.count(1)) # 男
list_num.append(gender.count(2)) # 女
attr = ["未知","男","女"]
pie = Pie("性别饼图",title_pos="center")
pie.add("", attr, list_num,is_label_show=True)
pie.render("sex_pie.html")
gendar=[]
for i in allData:
gendar.append(i['gendar'])
sexProcess(gendar)
# 评论者评分等级环状饼图
def scoreProcess(scores):
from pyecharts import Pie
list_num = []
list_num.append(scores.count(0))
list_num.append(scores.count(0.5))
list_num.append(scores.count(1))
list_num.append(scores.count(1.5))
list_num.append(scores.count(2))
list_num.append(scores.count(2.5))
list_num.append(scores.count(3))
list_num.append(scores.count(3.5))
list_num.append(scores.count(4))
list_num.append(scores.count(4.5))
list_num.append(scores.count(5))
attr = ["0", "0.5", "1","1.5","2","2.5", "3", "3.5","4","4.5","5"]
pie = Pie("评分等级环状饼图",title_pos="center")
pie.add("", attr, list_num, is_label_show=True,
label_text_color=None,
radius=[40, 75],
legend_orient="vertical",
legend_pos="left",
legend_top="100px",
center=[50,60]
)
pie.render("score_pie.html")
scores=[]
for i in allData:
scores.append(i['score'])
scoreProcess(scores)
# 观众分布图
def cityProcess(citysTotal):
from pyecharts import Geo
geo =Geo("《何以为家》观众分布", title_color='#fff', title_pos='center',
width=1200,height = 600, background_color = '#404a95')
attr, value = geo.cast(citysTotal)
geo.add("", attr, value, is_visualmap=True, visual_range=[0, 100], visual_text_color='#fff',
legend_pos = 'right', is_geo_effect_show = True, maptype='china',
symbol_size=10)
geo.render("city_geo.html")
# 城市名称的处理
citysTotal={}
coordinatesJson = pd.read_json('city_coordinates.json',encoding='utf-8')
for i in allData:
for j in coordinatesJson:
if str(i['cityName']) in str(j) :
if str(j) not in citysTotal:
citysTotal[str(j)]=1
else:
citysTotal[str(j)]=citysTotal[str(j)]+1
break
cityProcess(citysTotal)
posted on 2019-05-09 21:28 liliguang 阅读(2825) 评论(0) 编辑 收藏 举报
