Luminance

导航

数据采集与融合技术实践课第一次作业

第一题：软科2020中国大学主榜

代码与运行结果

下面代码可以独立运行。采集结果同时保存为CSV，原始网页响应和采集时间记录在 output/ 中。

"""题一：采集软科2020中国大学主榜并检查主榜完整性。"""
from datetime import datetime
from pathlib import Path
from urllib.parse import urljoin
import csv
import json
import logging
import re
import shutil
import subprocess
import sys

ROOT = Path(__file__).resolve().parent
OUT = ROOT / "output"
RAW = OUT / "raw"
OUT.mkdir(exist_ok=True)
RAW.mkdir(exist_ok=True)
sys.path.insert(0, str(ROOT / "_deps"))

import requests
from bs4 import BeautifulSoup

logging.basicConfig(
    filename=OUT / "作业日志.log",
    level=logging.INFO,
    encoding="utf-8",
    format="%(asctime)s %(levelname)s %(message)s",
)

URL = "https://www.shanghairanking.cn/rankings/bcur/2020"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                  "AppleWebKit/537.36 Chrome/131 Safari/537.36"
}
EXPECTED_ROWS = 567


def parse_payload(payload_path):
    """通过Node vm读取本地Nuxt回调状态，不在浏览器中执行页面代码。"""
    parser_path = RAW / "解析软科payload.js"
    parser_path.write_text(
        "const fs=require('fs'),vm=require('vm');let state;"
        "vm.runInNewContext(fs.readFileSync(process.argv[2],'utf8'),"
        "{__NUXT_JSONP__:(route,s)=>{state=s}}, {timeout:15000});"
        "const data=state&&state.data&&state.data[0]&&state.data[0].univData;"
        "if(!Array.isArray(data))throw Error('官方榜单数据数组不存在');"
        "process.stdout.write(JSON.stringify(data));",
        encoding="utf-8",
    )
    bundled_node = (
        Path.home()
        / ".cache/codex-runtimes/codex-primary-runtime/dependencies/node/bin/node.exe"
    )
    node_command = str(bundled_node) if bundled_node.exists() else shutil.which("node")
    if not node_command:
        raise RuntimeError("需要Node.js解析Nuxt数据；请安装Node.js并将node加入PATH")

    result = subprocess.run(
        [node_command, str(parser_path), str(payload_path)],
        capture_output=True,
        text=True,
        encoding="utf-8",
        timeout=30,
    )
    if result.returncode:
        raise RuntimeError(f"Nuxt payload解析失败：{result.stderr[-1000:]}")
    return json.loads(result.stdout)


def main():
    response = requests.get(URL, headers=HEADERS, timeout=30)
    response.raise_for_status()
    html_path = RAW / "软科2020榜单页面.html"
    html_path.write_bytes(response.content)

    soup = BeautifulSoup(response.content, "html.parser")
    table_rows = soup.select("table tr")
    dom_names = []
    for row in table_rows:
        cells = row.select("td")
        if len(cells) > 1:
            # 第二列除中文名外还可能带英文名和标签，下面用包含关系校验中文名。
            dom_names.append(cells[1].get_text(" ", strip=True))

    # HTML首屏只显示30所学校；用BeautifulSoup发现页面关联的官方Nuxt payload。
    script_sources = "\n".join(
        script.get("src", "") for script in soup.select("script[src]")
    )
    page_markup = response.text + "\n" + script_sources
    version = re.search(
        r"/_nuxt/static/(\d+)/rankings/bcur/2020/payload\.js", page_markup
    )
    if not version:
        raise RuntimeError("页面中未找到官方Nuxt payload.js链接")

    payload_url = urljoin(
        URL,
        f"/_nuxt/static/{version.group(1)}/rankings/bcur/2020/payload.js",
    )
    payload_response = requests.get(
        payload_url,
        headers={**HEADERS, "Referer": URL},
        timeout=30,
    )
    payload_response.raise_for_status()
    payload_path = RAW / "软科2020官方payload.js"
    payload_path.write_bytes(payload_response.content)
    records = parse_payload(payload_path)

    if len(records) != EXPECTED_ROWS:
        raise RuntimeError(
            f"完整性检查失败：主榜预期{EXPECTED_ROWS}条，实际{len(records)}条"
        )
    rows = [
        {
            "排名": record.get("ranking"),
            "学校名称": record.get("univNameCn"),
            "省市": record.get("province"),
            "学校类型": record.get("univCategory"),
            "总分": record.get("score"),
        }
        for record in records
    ]
    payload_names = [str(row["学校名称"]).strip() for row in rows[:len(dom_names)]]
    if len(dom_names) == 30:
        for dom_name, payload_name in zip(dom_names, payload_names):
            if re.sub(r"\s+", "", payload_name) not in re.sub(r"\s+", "", dom_name):
                raise RuntimeError("首屏DOM学校名称与payload数据不一致")

    collected_at = datetime.now().astimezone().isoformat(timespec="seconds")
    columns = ["排名", "学校名称", "省市", "学校类型", "总分"]
    csv_path = OUT / "软科2020中国大学主榜.csv"
    with csv_path.open("w", encoding="utf-8-sig", newline="") as file:
        writer = csv.DictWriter(file, fieldnames=columns)
        writer.writeheader()
        writer.writerows(rows)

    metadata = {
        "source_url": response.url,
        "data_url": payload_url,
        "collected_at": collected_at,
        "parsed_rows": len(rows),
        "html_dom_tr_rows": len(table_rows),
        "note": "DOM为首屏分页；CSV来自页面引用的完整官方Nuxt payload。",
    }
    metadata_path = OUT / "软科2020采集元数据.json"
    metadata_path.write_text(
        json.dumps(metadata, ensure_ascii=False, indent=2), encoding="utf-8"
    )

    print(" | ".join(columns))
    for row in rows[:10]:
        print(" | ".join(str(row[column]) for column in columns))
    print(
        f"完整榜单：{len(rows)}条；首屏DOM含表头共{len(table_rows)}行；"
        f"来源时间：{collected_at}"
    )
    logging.info("ranking ok rows=%d payload=%s", len(rows), payload_url)


if __name__ == "__main__":
    main()

本次运行结果：

  • 来源：https://www.shanghairanking.cn/rankings/bcur/2020
  • 采集时间：2026-10-20T21:01:32+08:00
  • 主榜记录：567 条，排名覆盖1至567；首屏HTML表格有 31 行（含表头），可见30所。
  • CSV首条：清华大学（852.5分）；末条：首钢工学院（55.9分）。
  • BeautifulSoup检查首屏，之后读取同一页面引用的官方Nuxt payload；Node仅解析本地保存的JS数据。CSV前五条：1 清华大学（北京，综合，852.5）；2 北京大学（北京，综合，746.7）；3 浙江大学（浙江，综合，649.2）；4 上海交通大学（上海，综合，625.9）；5 南京大学（江苏，综合，566.1）

截图

1

心得体会

页面首屏表格为31行，包含表头和30所大学；表格第二列还混有英文名和标签，不能直接把单元格整段文字当作中文校名。程序从单元格中核对payload校名，而全榜记录取自该页声明的Nuxt payload。最终记录覆盖排名1到567，首位清华大学852.5分，末位首钢工学院55.9分，首屏校名顺序与payload一致。这样校验的是来源提供的全量数据，而不是把首屏30行误写成全榜。

第二题：当当网书包搜索与比价

代码与运行结果

下面代码可以独立运行。采集结果同时保存为CSV，原始网页响应和采集时间记录在 output/ 中。

"""题二：requests+re抓取当当网“书包”公开搜索页并按展示价排序。"""
from pathlib import Path
from datetime import datetime
from urllib.parse import urljoin
import csv
import html
import json
import logging
import re
import sys

ROOT = Path(__file__).resolve().parent
OUT = ROOT / "output"
RAW = OUT / "raw"
OUT.mkdir(exist_ok=True)
RAW.mkdir(exist_ok=True)
sys.path.insert(0, str(ROOT / "_deps"))
import requests

logging.basicConfig(filename=OUT / "作业日志.log", level=logging.INFO, encoding="utf-8",
                    format="%(asctime)s %(levelname)s %(message)s")
URL = "https://search.dangdang.com/?key=%CA%E9%B0%FC&act=input"
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131 Safari/537.36"}

def clean_text(value):
    value = re.sub(r"<[^>]+>", "", value)
    return re.sub(r"\s+", " ", html.unescape(value)).strip()

def main():
    response = requests.get(URL, headers=HEADERS, timeout=30)
    response.raise_for_status()
    (RAW / "当当书包搜索页.html").write_bytes(response.content)
    source = response.content.decode("gb18030", errors="replace")
    cards = re.findall(
        r"<li\b[^>]*class=[\"']line\d+[\"'][^>]*>.*?</li\s*>",
        source,
        flags=re.I | re.S,
    )
    rows = []
    for card in cards:
        name_block = re.search(r'<p\s+class=["\']name["\'][^>]*>(.*?)</p\s*>', card, flags=re.I | re.S)
        title_match = None
        if name_block:
            anchor = re.search(r'<a\b([^>]*)>', name_block.group(1), flags=re.I | re.S)
            if anchor:
                attrs = anchor.group(1)
                title_attr = re.search(r'\btitle\s*=\s*(["\'])(.*?)\1', attrs, flags=re.I | re.S)
                href_attr = re.search(r'\bhref\s*=\s*(["\'])(.*?)\1', attrs, flags=re.I | re.S)
                if title_attr and href_attr:
                    title_match = (title_attr.group(2), href_attr.group(2))
        price_match = re.search(r'<span\s+class=["\']price_n["\'][^>]*>\s*([^<]+)', card, flags=re.I | re.S)
        if not title_match or not price_match:
            continue
        price_text = clean_text(price_match.group(1)).replace(",", "")
        amount = re.search(r"\d+(?:\.\d{1,2})?", price_text)
        if not amount:
            continue
        name = clean_text(title_match[0])
        if "随书包含" in name or not any(word in name for word in ("书包", "双肩包", "背包")):
            continue
        rows.append({"商品名称": name,
                     "价格(元)": float(amount.group()),
                     "商品链接": urljoin(response.url, html.unescape(title_match[1]))})
    if not rows:
        raise RuntimeError("页面未解析到商品和价格；原始响应已保存供检查")
    rows.sort(key=lambda row: row["价格(元)"])
    now = datetime.now().astimezone().isoformat(timespec="seconds")
    with (OUT / "当当书包比价.csv").open("w", encoding="utf-8-sig", newline="") as f:
        writer = csv.DictWriter(f, fieldnames=["商品名称", "价格(元)", "商品链接"])
        writer.writeheader()
        writer.writerows(rows)
    count_match = re.search(
        r"<span\s+class=[\"']sp total[\"'][^>]*>\s*共.*?<em[^>]*>([\d,]+)</em>\s*件商品",
        source,
        flags=re.I | re.S,
    )
    meta = {"source_url": response.url, "keyword": "书包", "collected_at": now,
            "product_cards_found": len(cards), "page_items_parsed": len(rows),
            "excluded_cards": len(cards) - len(rows),
            "page_total_text": f"共 {count_match.group(1)} 件商品" if count_match else "页面总量文字未识别",
            "price_definition": "页面搜索结果卡片展示价；未进入详情或结算，不含运费及促销券核验。",
            "scope": "当当网该关键词搜索结果当前页可解析条目"}
    (OUT / "当当书包采集元数据.json").write_text(json.dumps(meta, ensure_ascii=False, indent=2), encoding="utf-8")
    print("序号 | 价格(元) | 商品名称")
    for i, row in enumerate(rows[:15], 1):
        print(f"{i:02d} | {row['价格(元)']:.2f} | {row['商品名称']}")
    print(f"共解析当前页 {len(rows)} 条；最低展示价 ¥{rows[0]['价格(元)']:.2f}；采集时间 {now}")
    logging.info("shop ok rows=%d url=%s", len(rows), response.url)

if __name__ == "__main__":
    main()

本次运行结果：

  • 来源：https://search.dangdang.com/?key=����&act=input
  • 采集时间：2026-10-20T21:01:33+08:00
  • 搜索页总量文字：共 31543 件商品；从当前页 60 个卡片中筛出 57 条名称含书包、双肩包或背包的商品，排除 3 条。
  • 按展示价升序排列，最低为 ¥36.0：大的一只帆布包单肩包学生书包百搭大包包环保帆布袋 春夏季女包。这个商品是帆布单肩包，标题含“学生书包”；搜索结果品类混杂，因此最低价不等于同型号书包中最优惠。完整当前页清单见 output/当当书包比价.csv。
  • 前15条已在控制台输出。展示价未核对详情页、运费、优惠券或结算价。

截图位置：

2

心得体会

当当响应采用GB18030编码。正则用DOTALL跨行读取卡片，并分别找出名称区a标签中的title和href，因此两种属性的先后顺序都可处理；随后提取price_n中的数字、解码HTML实体并转成数值排序。当前页抓到60个商品卡片，排除3个名称不含目标包类关键词的结果，留下57条。标题含‘随书包含’的图书会被‘书包’子串误命中，过滤步骤解决了这类误收。最低36元是一只帆布单肩包，不能拿来和其他款式直接比较性价比；页面展示价也不等于结算价。

第三题：新闻页图片采集

代码与运行结果

下面代码可以独立运行。采集结果同时保存为CSV，原始网页响应和采集时间记录在 output/ 中。

"""题三：下载福州大学新闻页直接引用的JPG/JPEG/PNG图片并校验格式。"""
from datetime import datetime
from hashlib import sha1
from io import BytesIO
from pathlib import Path
from urllib.parse import urljoin, urlparse
import csv
import html
import json
import logging
import re
import sys
import time

ROOT = Path(__file__).resolve().parent
OUT = ROOT / "output"
RAW = OUT / "raw"
IMAGE_DIR = OUT / "assets" / "news_images"
for directory in (OUT, RAW, IMAGE_DIR):
    directory.mkdir(parents=True, exist_ok=True)

sys.path.insert(0, str(ROOT / "_deps"))
import requests
from bs4 import BeautifulSoup
from PIL import Image

logging.basicConfig(
    filename=OUT / "作业日志.log",
    level=logging.INFO,
    encoding="utf-8",
    format="%(asctime)s %(levelname)s %(message)s",
)

PAGE_URL = "https://news.fzu.edu.cn/yxfd.htm"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                  "AppleWebKit/537.36 Chrome/131 Safari/537.36"
}
IMAGE_ATTRIBUTES = ("src", "data-src", "data-original", "data-lazy-src", "data-echo")


def collect_image_urls(soup, page_url):
    """从HTML图片标签、图片链接、srcset和行内CSS中提取去重后的图片URL。"""
    urls = []

    def add_candidate(raw_url):
        raw_url = html.unescape(str(raw_url).strip().strip("\"'"))
        if not raw_url or raw_url.startswith(("data:", "javascript:", "#")):
            return
        absolute_url = urljoin(page_url, raw_url)
        extension = Path(urlparse(absolute_url).path).suffix.lower()
        if extension in (".jpg", ".jpeg", ".png") and absolute_url not in urls:
            urls.append(absolute_url)

    for tag in soup.find_all(("img", "source", "a")):
        if tag.name == "a" and tag.get("href"):
            add_candidate(tag["href"])
        for attribute in IMAGE_ATTRIBUTES:
            if tag.get(attribute):
                add_candidate(tag[attribute])
        srcset = tag.get("srcset", "")
        for candidate in srcset.split(","):
            parts = candidate.strip().split()
            if parts:
                add_candidate(parts[0])

    # 背景图片可能出现在任意带style属性的元素或<style>规则中。
    for tag in soup.select("[style]"):
        style = tag.get("style", "")
        for match in re.findall(r"url\(\s*[\"']?([^\"')]+)", style, flags=re.I):
            add_candidate(match)
    for style_tag in soup.find_all("style"):
        for match in re.findall(
            r"url\(\s*[\"']?([^\"')]+)", style_tag.get_text(), flags=re.I
        ):
            add_candidate(match)
    return urls


def main():
    session = requests.Session()
    page = session.get(PAGE_URL, headers=HEADERS, timeout=30)
    page.raise_for_status()
    (RAW / "福州大学新闻网页面.html").write_bytes(page.content)
    soup = BeautifulSoup(page.content, "html.parser")
    image_urls = collect_image_urls(soup, page.url)

    # 清除上次程序生成的同类文件，避免旧下载混入本次截图和清单。
    for pattern in ("*_??????????.jpg", "*_??????????.png"):
        for old_file in IMAGE_DIR.glob(pattern):
            old_file.unlink()

    rows = []
    failures = []
    for index, image_url in enumerate(image_urls, start=1):
        try:
            if index > 1:
                time.sleep(0.4)
            response = session.get(
                image_url,
                headers={**HEADERS, "Referer": page.url},
                timeout=25,
            )
            response.raise_for_status()
            with Image.open(BytesIO(response.content)) as image:
                image.verify()
            with Image.open(BytesIO(response.content)) as image:
                image_format = image.format
                width, height = image.size
            if image_format not in ("JPEG", "PNG"):
                raise ValueError(f"图片实际格式为{image_format}，不属于JPG/PNG")

            extension = ".jpg" if image_format == "JPEG" else ".png"
            suffix = sha1(image_url.encode("utf-8")).hexdigest()[:10]
            file_path = IMAGE_DIR / f"{index:03d}_{suffix}{extension}"
            file_path.write_bytes(response.content)
            rows.append({
                "本地相对路径": file_path.relative_to(ROOT).as_posix(),
                "图片URL": image_url,
                "格式": image_format,
                "宽度": width,
                "高度": height,
                "字节数": len(response.content),
            })
        except Exception as error:
            failures.append({"图片URL": image_url, "错误": str(error)[:300]})
            logging.warning("image failed %s: %s", image_url, error)

    collected_at = datetime.now().astimezone().isoformat(timespec="seconds")
    csv_path = OUT / "福州大学新闻图片.csv"
    with csv_path.open("w", encoding="utf-8-sig", newline="") as file:
        columns = ["本地相对路径", "图片URL", "格式", "宽度", "高度", "字节数"]
        writer = csv.DictWriter(file, fieldnames=columns)
        writer.writeheader()
        writer.writerows(rows)

    metadata = {
        "source_page": page.url,
        "collected_at": collected_at,
        "discovered_image_urls": len(image_urls),
        "downloaded_and_format_verified": len(rows),
        "failed": len(failures),
        "failures": failures,
        "scope": "指定新闻页HTML直接暴露的图片、图片链接、srcset和行内CSS引用；"
                 "不递归抓取外部CSS文件及后续分页。",
    }
    metadata_path = OUT / "新闻图片采集记录.json"
    metadata_path.write_text(
        json.dumps(metadata, ensure_ascii=False, indent=2), encoding="utf-8"
    )

    print(
        f"页面发现JPG/JPEG/PNG候选 {len(image_urls)} 个；"
        f"实际下载且格式校验成功 {len(rows)} 个；失败 {len(failures)} 个；"
        f"采集时间 {collected_at}"
    )
    for row in rows[:8]:
        print(f"{row['本地相对路径']} | {row['格式']} {row['宽度']}x{row['高度']}")
    if failures:
        print("失败明细见 output/新闻图片采集记录.json")
    logging.info(
        "images discovered=%d success=%d failed=%d",
        len(image_urls), len(rows), len(failures),
    )
    if failures or not rows:
        raise RuntimeError("图片下载未全部成功或未发现图片，请查看采集记录JSON")


if __name__ == "__main__":
    main()

本次运行结果：

  • 来源页面：https://news.fzu.edu.cn/yxfd.htm
  • 采集时间：2026-10-20T21:01:46+08:00
  • 页面HTML内发现 19 个候选URL，下载且用Pillow验证实际格式成功 19 张，失败 0 张。
  • 图片文件位于 output/assets/news_images/；部分文件示例：output/assets/news_images/001_3973f4aa4d.png、output/assets/news_images/002_46e9ca3cc3.png、output/assets/news_images/003_f761c1577c.png、output/assets/news_images/004_8c87d0a22e.png、output/assets/news_images/005_47c8d4aadc.png。采集范围是当前HTML直接暴露的图片、图片链接、srcset和行内CSS引用，不递归抓取外部CSS文件及后续分页。

截图位置：

3

4

心得体会

图片可能在src、懒加载属性、srcset、任意元素的行内style、style标签或图片链接中；程序把相对路径交给urljoin补成绝对URL，并按URL去重。保存文件名由序号和URL哈希组成，避免不同来源同名覆盖。此次发现19个候选，成功下载并验证19张，失败0张；清单中既有新闻配图，也有页面图标等资源，并非全是新闻照片。Pillow按内容验证JPEG/PNG实际格式，结果和失败原因写入CSV与JSON。采集范围是当前HTML直接暴露的图片、图片链接、srcset和行内CSS引用，不递归外部CSS文件及后续分页。

posted on 2026-10-20 21:41  Luminance  阅读(6)  评论(0)    收藏  举报