2024/5/21

豆瓣图书评论数据爬取。以《平凡的世界》、《都挺好》等为分析对象，编写程序爬取豆瓣读书上针对该图书的短评信息，要求：

（1）对前3页短评信息进行跨页连续爬取；

（2）爬取的数据包含用户名、短评内容、评论时间、评分和点赞数（有用数）；

（3）能够根据选择的排序方式（热门或最新）进行爬取，并分别针对热门和最新排序，输出前10位短评信息（包括用户名、短评内容、评论时间、评分和点赞数）。

（4）根据点赞数的多少，按照从多到少的顺序将排名前10位的短评信息输出；

import re
from collections import Counter
import requests
from lxml import etree
import pandas as pd
import jieba
import matplotlib.pyplot as plt
from wordcloud import WordCloud

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36 Edg/101.0.1210.39"
}

comments_list = []
words_list = []

def clean_text(text):
    text = re.sub(r"[\n”“|,，；;''/?! 。的了是]", "", text)
    text = re.sub(r"^\d+::", "", text)
    text = re.sub(
        r"(https?://)?([a-zA-Z0-9]+)(\.[a-zA-Z0-9]+)(\.[a-zA-Z0-9]+)*(/[a-zA-Z0-9]+)*", "", text, re.IGNORECASE)
    text = re.sub(u"年|月|日|周一|周二|周三|周四|周五|周六", "", text)
    text = re.sub(r"[^a-zA-Z]\d+", "", text)
    text = re.sub(r"\s+", "", text)
    return text

def extract_comments(url):
    resp = requests.get(url, headers=headers).text
    html = etree.HTML(resp)
    comment_list = html.xpath(".//div[@class='comment']")
    for comment in comment_list:
        username = comment.xpath(".//span[@class='comment-info']/a/text()")[0]
        content = comment.xpath(".//p[@class='comment-content']/span[@class='short']/text()")[0].strip()
        words_list.extend([clean_text(word) for word in jieba.cut(content, cut_all=False, HMM=False) if len(clean_text(word)) >= 2])
        time = comment.xpath(".//span[@class='comment-info']/a/text()")[1]
        mark = comment.xpath(".//span[@class='comment-info']/span/@title")
        score = 0 if len(mark) == 0 else 5 if mark[0] == "力荐" else 4 if mark[0] == "推荐" else 3 if mark[0] == "还行" else 2 if mark[0] == "较差" else 1
        likes = int(comment.xpath(".//span[@class='comment-vote']/span[@class='vote-count']/text()")[0])
        comments_list.append([username, content, time, score, likes])

def generate_wordcloud(words):
    text = ' '.join(words)
    wordcloud = WordCloud(background_color='white', width=1000, height=700, font_path='simhei.ttf', margin=10).generate(text)

    plt.imshow(wordcloud)
    plt.axis("off")
    plt.show()
    wordcloud.to_file('wordcloud.png')

print("请选择以下选项:")
print("   1.热门评论")
print("   2.最新评论")
selected_option = int(input())

if selected_option == 1 or selected_option == 2:
    comments_list.clear()
    words_list.clear()

    for i in range(0, 60, 20):
        sort_type = "new_score" if selected_option == 1 else "time"
        url = f"https://book.douban.com/subject/10517238/comments/?start={i}&limit=20&status=P&sort={sort_type}"
        extract_comments(url)

    columns = ['用户名', '短评内容', '评论时间', '评分', '点赞数']
    df = pd.DataFrame(comments_list, columns=columns)

    print("前10位短评信息：")
    print(df.head(10))

    print("点赞数前10位的短评信息：")
    df_sorted = df.sort_values(by='点赞数', ascending=False)
    print(df_sorted.head(10))

    generate_wordcloud(words_list)
else:
    print("无效选项，请重新运行程序并选择1或2。")