数据采集第三次作业
数据采集第三次作业
102402132 邱福铭
运行环境:Windows 11、Python 3.14、Scrapy 2.20、requests、BeautifulSoup、PyMySQL
数据库:MySQLassignment3(本地用 MariaDB 11.4 验证)
Gitee 作业3 文件夹:
https://gitee.com/qiu-fuming/2026shujucaijiyuronghejishu/tree/master/作业3
一、作业①②:图片爬虫与股票行情
1)实验实现
作业①:中国新闻网图片频道,单线程与多线程
选择中国新闻网图片频道 https://www.chinanews.com.cn/photo/ 作为目标网站。它每天更新图片专题,
详情页数量足够,而且页面里的 <img> 标签结构清晰,适合练习“解析图片 URL + 下载”的完整流程。
程序分三步:
- 从图片频道首页发现
/tp/hd2011/形式的详情页,取前 32 个; - 逐页用 BeautifulSoup 解析
<img>,把相对地址补全为绝对地址,按 URL 去重; - 截取前 132 个 URL,分别用单线程和多线程下载到
images/single、images/multi。
关键代码:
"""第三次作业①:中国新闻网图片频道,单线程 / 多线程图片爬虫。
学号 102402132:
- 总页数取学号最后 2 位:32 页;
- 总下载图片数取学号最后 3 位:132 张。
抓取对象是中国新闻网图片频道的 32 个图片详情页,解析页面中的
<img> 标签,去重后下载前 132 张图片,并把每张图片的 URL 输出到控制台。
用法:
py -X utf8 .\1_image_spider.py --mode single
py -X utf8 .\1_image_spider.py --mode multi
"""
from __future__ import annotations
import argparse
import hashlib
import json
import re
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import dataclass
from pathlib import Path
from urllib.parse import urljoin, urlparse
import requests
from bs4 import BeautifulSoup
# 学号 102402132 的控制参数
MAX_PAGES = 32
MAX_IMAGES = 132
MAX_WORKERS = 8
SEED_URL = "https://www.chinanews.com.cn/photo/"
BASE_DIR = Path(__file__).resolve().parent
OUTPUT_DIR = BASE_DIR / "output"
HEADERS = {
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/131.0.0.0 Safari/537.36"
),
"Referer": SEED_URL,
}
IMAGE_EXT_RE = re.compile(r"\.(?:jpe?g|png|webp|gif|bmp)(?:$|\?)", re.I)
@dataclass
class ImageTask:
"""一张待下载图片的信息。"""
index: int
url: str
def fetch_html(url: str, session: requests.Session, retries: int = 2) -> str:
"""带重试地下载网页 HTML。"""
last_error: Exception | None = None
for attempt in range(retries + 1):
try:
response = session.get(url, headers=HEADERS, timeout=20)
response.raise_for_status()
response.encoding = response.apparent_encoding or "utf-8"
return response.text
except Exception as exc: # noqa: BLE001 - 网络异常统一重试
last_error = exc
if attempt < retries:
time.sleep(1 + attempt)
raise RuntimeError(f"页面请求失败:{url};{last_error}")
def collect_article_urls(session: requests.Session) -> list[str]:
"""从图片频道首页发现图片详情页,最多取 32 个。"""
html = fetch_html(SEED_URL, session)
soup = BeautifulSoup(html, "html.parser")
urls: list[str] = []
for anchor in soup.find_all("a", href=True):
url = urljoin(SEED_URL, anchor["href"])
# 图片详情页的固定路径特征
if "/tp/hd2011/" in url and url.endswith(".shtml") and url not in urls:
urls.append(url)
if len(urls) >= MAX_PAGES:
break
if len(urls) < MAX_PAGES:
raise RuntimeError(f"只发现 {len(urls)} 个详情页,少于要求的 {MAX_PAGES} 页")
return urls
def is_image_url(url: str) -> bool:
"""过滤 data URI、空地址和非图片扩展名。"""
if not url or url.startswith("data:"):
return False
return bool(IMAGE_EXT_RE.search(urlparse(url).path + urlparse(url).query))
def collect_image_urls(session: requests.Session, article_urls: list[str]) -> list[str]:
"""逐页解析 <img>,按 URL 去重后返回最多 132 张图片地址。"""
image_urls: list[str] = []
seen: set[str] = set()
for page_no, article_url in enumerate(article_urls, 1):
try:
html = fetch_html(article_url, session)
except Exception as exc: # noqa: BLE001 - 单页失败不影响其他页
print(f"[页面失败] 第 {page_no:02d}/{MAX_PAGES} 页 {article_url}:{exc}")
continue
soup = BeautifulSoup(html, "html.parser")
page_count = 0
for img in soup.find_all("img"):
src = img.get("src") or img.get("data-src") or img.get("data-original") or ""
src = src.strip()
if not src:
continue
url = urljoin(article_url, src)
if not is_image_url(url) or url in seen:
continue
seen.add(url)
image_urls.append(url)
page_count += 1
print(f"[解析] 第 {page_no:02d}/{MAX_PAGES} 页:新增 {page_count} 张,累计 {len(image_urls)} 张")
if len(image_urls) < MAX_IMAGES:
raise RuntimeError(f"只解析到 {len(image_urls)} 张图片,少于要求的 {MAX_IMAGES} 张")
return image_urls[:MAX_IMAGES]
def extension_for(url: str, content_type: str) -> str:
"""根据 Content-Type 和 URL 选择图片扩展名。"""
content_type = (content_type or "").lower()
mapping = {
"image/jpeg": ".jpg",
"image/jpg": ".jpg",
"image/png": ".png",
"image/webp": ".webp",
"image/gif": ".gif",
"image/bmp": ".bmp",
}
for key, ext in mapping.items():
if key in content_type:
return ext
suffix = Path(urlparse(url).path).suffix.lower()
return suffix if suffix in {".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp"} else ".jpg"
def download_one(task: ImageTask, target_dir: Path) -> tuple[ImageTask, Path | None, str]:
"""下载单张图片,返回任务、文件路径和错误信息。"""
try:
response = requests.get(task.url, headers=HEADERS, timeout=30)
response.raise_for_status()
content_type = response.headers.get("Content-Type", "")
if not content_type.lower().startswith("image/"):
return task, None, f"不是图片:{content_type}"
ext = extension_for(task.url, content_type)
digest = hashlib.md5(task.url.encode("utf-8")).hexdigest()[:8]
path = target_dir / f"{task.index:03d}_{digest}{ext}"
path.write_bytes(response.content)
return task, path, ""
except Exception as exc: # noqa: BLE001 - 单张失败记录后继续
return task, None, str(exc)
def run(mode: str) -> None:
"""执行一次完整抓取:32 页,下载 132 张图片。"""
target_dir = BASE_DIR / "images" / ("single" if mode == "single" else "multi")
target_dir.mkdir(parents=True, exist_ok=True)
OUTPUT_DIR.mkdir(exist_ok=True)
session = requests.Session()
started = time.perf_counter()
print(f"模式:{'单线程' if mode == 'single' else '多线程'}")
print(f"限制:总页数 {MAX_PAGES},总图片数 {MAX_IMAGES}")
article_urls = collect_article_urls(session)
print(f"已发现详情页:{len(article_urls)} 个")
image_urls = collect_image_urls(session, article_urls)
print(f"去重后图片 URL:{len(image_urls)} 张,开始下载……\n")
tasks = [ImageTask(i, url) for i, url in enumerate(image_urls, 1)]
results: list[tuple[ImageTask, Path | None, str]] = []
if mode == "single":
for task in tasks:
result = download_one(task, target_dir)
results.append(result)
path, error = result[1], result[2]
if path:
print(f"[{task.index:03d}/{MAX_IMAGES}] {task.url} -> {path.name}")
else:
print(f"[{task.index:03d}/{MAX_IMAGES}] 失败 {task.url}:{error}")
else:
with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
futures = {executor.submit(download_one, task, target_dir): task for task in tasks}
for done, future in enumerate(as_completed(futures), 1):
result = future.result()
results.append(result)
task, path, error = result
if path:
print(f"[{done:03d}/{MAX_IMAGES}] {task.url} -> {path.name}")
else:
print(f"[{done:03d}/{MAX_IMAGES}] 失败 {task.url}:{error}")
downloaded = sum(1 for _, path, _ in results if path)
failed = len(results) - downloaded
elapsed = time.perf_counter() - started
report = {
"mode": mode,
"max_pages": MAX_PAGES,
"max_images": MAX_IMAGES,
"article_pages": len(article_urls),
"image_urls": len(image_urls),
"downloaded": downloaded,
"failed": failed,
"elapsed_seconds": round(elapsed, 2),
"output_dir": str(target_dir),
}
(OUTPUT_DIR / f"image_report_{mode}.json").write_text(
json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8"
)
(OUTPUT_DIR / f"image_urls_{mode}.txt").write_text(
"\n".join(url for url in image_urls) + "\n", encoding="utf-8"
)
print("\n" + "=" * 72)
print(f"完成:{mode},页数 {len(article_urls)},URL {len(image_urls)},"
f"下载 {downloaded},失败 {failed},耗时 {elapsed:.2f} 秒")
print(f"保存目录:{target_dir}")
print("=" * 72)
def main() -> None:
parser = argparse.ArgumentParser(description="中国新闻网图片爬虫(单线程/多线程)")
parser.add_argument("--mode", choices=["single", "multi"], default="single")
args = parser.parse_args()
run(args.mode)
if __name__ == "__main__":
main()
单线程和多线程的对比:
| 方式 | 页数 | 图片 URL | 成功下载 | 失败 | 耗时 |
|---|---|---|---|---|---|
| 单线程 | 32 | 132 | 132 | 0 | 110.03 秒 |
| 多线程(8 线程) | 32 | 132 | 132 | 0 | 54.59 秒 |
多线程把下载阶段放进 ThreadPoolExecutor(max_workers=8),页面解析仍按顺序执行。因为瓶颈在图片
网络 I/O,不在解析,所以 8 个下载线程能把时间压到单线程的一半左右。

图 1 单线程运行结果:32 页、132 个 URL、132 张图片

图 2 多线程运行结果:同样 132 张,耗时 54.59 秒
Gitee 作业①:
https://gitee.com/qiu-fuming/2026shujucaijiyuronghejishu/tree/master/作业3
作业②:东方财富沪深 A 股,Scrapy + Item + Pipeline + MySQL
股票数据来自东方财富行情接口:
https://push2.eastmoney.com/api/qt/clist/get
接口用 pn 控制页码、pz=100 控制每页数量,fs 指定沪深 A 股范围,fid=f3 按涨跌幅排序。
data.total 给出总数,程序据此生成后续页请求。股票接口返回 JSON,所以这一题用 Scrapy 的
json.loads 解析字段;外汇题再用 XPath 解析 HTML 表格,两个 spider 共用 Item/Pipeline/MySQL 的路线。
items.py 中定义 StockItem,字段和作业表格一一对应:
| Item 字段 | MySQL 列 | 含义 |
|---|---|---|
bStockNo |
bStockNo |
股票代码 |
bStockName |
bStockName |
股票名称 |
latestPrice |
latestPrice |
最新报价 |
changePercent |
changePercent |
涨跌幅 |
changeAmount |
changeAmount |
涨跌额 |
volume |
volume |
成交量(万/亿) |
turnover |
turnover |
成交额(万/亿) |
amplitude |
amplitude |
振幅 |
high / low |
high / low |
最高 / 最低 |
openPrice / prevClose |
openPrice / prevClose |
今开 / 昨收 |
股票 Spider 的核心代码:
"""东方财富沪深 A 股行情 Spider。"""
from __future__ import annotations
import json
import math
from urllib.parse import urlencode
import scrapy
from scrapy import signals
from demo.items import StockItem
class StockSpider(scrapy.Spider):
"""按页请求东方财富行情接口,Item 交给 MySQL Pipeline。"""
name = "stock"
allowed_domains = ["eastmoney.com"]
api_url = "https://push2.eastmoney.com/api/qt/clist/get"
page_size = 100
max_pages = 100
custom_settings = {
"DOWNLOAD_DELAY": 0.08,
"CONCURRENT_REQUESTS": 6,
}
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.total = 0
self.first_page = True
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
spider = super().from_crawler(crawler, *args, **kwargs)
crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
return spider
async def start(self):
"""Scrapy 2.20 的入口;旧版本可继续使用 start_requests。"""
yield self.make_request(1)
def start_requests(self):
yield self.make_request(1)
def make_request(self, page: int) -> scrapy.Request:
"""构造一页行情接口请求。"""
params = {
"pn": page,
"pz": self.page_size,
"po": 1,
"np": 1,
"ut": "bd1d9ddb04089700cf9c27f6f7426281",
"fltt": 2,
"invt": 2,
"fid": "f3",
"fs": "m:0+t:6,m:0+t:80,m:1+t:2,m:1+t:23,m:0+t:81+s:2048",
"fields": "f2,f3,f4,f5,f6,f7,f12,f14,f15,f16,f17,f18",
}
return scrapy.Request(
f"{self.api_url}?{urlencode(params)}",
callback=self.parse,
headers={
"Referer": "https://quote.eastmoney.com/",
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/131.0.0.0 Safari/537.36"
),
},
)
@staticmethod
def number(value):
"""接口里的 '-' 转为 None,避免 MySQL DECIMAL 报错。"""
try:
return float(value)
except (TypeError, ValueError):
return None
@staticmethod
def human_number(value, unit: str = "") -> str:
"""把成交量/成交额格式化为万、亿,便于和作业表格对照。"""
try:
number = float(value)
except (TypeError, ValueError):
return "-"
if abs(number) >= 100_000_000:
return f"{number / 100_000_000:.2f}亿{unit}"
if abs(number) >= 10_000:
return f"{number / 10_000:.2f}万{unit}"
return f"{number:.0f}{unit}"
def parse(self, response):
"""解析 JSON,首页同时生成后续分页请求。"""
try:
payload = json.loads(response.text)
except json.JSONDecodeError:
self.logger.error("股票接口不是合法 JSON:%s", response.url)
return
data = payload.get("data") or {}
self.total = int(data.get("total") or 0)
rows = data.get("diff") or []
if isinstance(rows, dict):
rows = list(rows.values())
if self.first_page and self.total:
self.first_page = False
pages = min(math.ceil(self.total / self.page_size), self.max_pages)
self.logger.info("接口总数 %s,共 %s 页", self.total, pages)
for page in range(2, pages + 1):
yield self.make_request(page)
for row in rows:
code = str(row.get("f12") or "").strip()
if not code:
continue
yield StockItem(
bStockNo=code,
bStockName=row.get("f14") or "",
latestPrice=self.number(row.get("f2")),
changePercent=self.number(row.get("f3")),
changeAmount=self.number(row.get("f4")),
volume=self.human_number(row.get("f5")),
turnover=self.human_number(row.get("f6")),
amplitude=self.number(row.get("f7")),
high=self.number(row.get("f15")),
low=self.number(row.get("f16")),
openPrice=self.number(row.get("f17")),
prevClose=self.number(row.get("f18")),
)
def spider_closed(self, spider, reason):
"""日志中输出总数,便于截图。"""
stats = self.crawler.stats
count = stats.get_value("item_scraped_count", 0) if stats else 0
print(f"[完成] 股票接口总数={self.total},Item 提交={count},原因={reason}")
Pipeline 按 Item 类型分流,股票每 200 条批量 executemany,用 ON DUPLICATE KEY UPDATE 保证重复运行
不产生重复股票代码。这样既保留 Item/Pipeline 的序列化路线,也避免 5000 多条逐条 commit:
"""把 StockItem / ForexItem 序列化写入 MySQL。"""
from __future__ import annotations
import re
import pymysql
from itemadapter import ItemAdapter
from demo.items import ForexItem, StockItem
class MySQLPipeline:
"""批量写入 MySQL,股票和外汇分别落到两张表。"""
stock_sql = """
INSERT INTO stocks (
bStockNo, bStockName, latestPrice, changePercent, changeAmount,
volume, turnover, amplitude, high, low, openPrice, prevClose
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
ON DUPLICATE KEY UPDATE
bStockName = VALUES(bStockName),
latestPrice = VALUES(latestPrice),
changePercent = VALUES(changePercent),
changeAmount = VALUES(changeAmount),
volume = VALUES(volume),
turnover = VALUES(turnover),
amplitude = VALUES(amplitude),
high = VALUES(high),
low = VALUES(low),
openPrice = VALUES(openPrice),
prevClose = VALUES(prevClose),
collected_at = CURRENT_TIMESTAMP
"""
forex_sql = """
INSERT INTO forex (
currency, tbp, cbp, tsp, csp, quoteDate, quoteTime
) VALUES (%s, %s, %s, %s, %s, %s, %s)
ON DUPLICATE KEY UPDATE
tbp = VALUES(tbp),
cbp = VALUES(cbp),
tsp = VALUES(tsp),
csp = VALUES(csp),
quoteDate = VALUES(quoteDate),
quoteTime = VALUES(quoteTime),
collected_at = CURRENT_TIMESTAMP
"""
stock_ddl = """
CREATE TABLE IF NOT EXISTS stocks (
id INT NOT NULL AUTO_INCREMENT,
bStockNo VARCHAR(16) NOT NULL,
bStockName VARCHAR(64),
latestPrice DECIMAL(14, 4),
changePercent DECIMAL(10, 4),
changeAmount DECIMAL(14, 4),
volume VARCHAR(32),
turnover VARCHAR(32),
amplitude DECIMAL(10, 4),
high DECIMAL(14, 4),
low DECIMAL(14, 4),
openPrice DECIMAL(14, 4),
prevClose DECIMAL(14, 4),
collected_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (id),
UNIQUE KEY uk_stock_no (bStockNo)
) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4
"""
forex_ddl = """
CREATE TABLE IF NOT EXISTS forex (
id INT NOT NULL AUTO_INCREMENT,
currency VARCHAR(64) NOT NULL,
tbp DECIMAL(12, 4),
cbp DECIMAL(12, 4),
tsp DECIMAL(12, 4),
csp DECIMAL(12, 4),
quoteDate VARCHAR(32),
quoteTime VARCHAR(32),
collected_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (id),
UNIQUE KEY uk_currency (currency)
) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4
"""
def __init__(self, config: dict):
self.config = config
self.connection = None
self.stock_rows: list[tuple] = []
self.forex_rows: list[tuple] = []
self.stock_count = 0
self.forex_count = 0
@classmethod
def from_crawler(cls, crawler):
return cls(crawler.settings.getdict("MYSQL_CONFIG"))
def open_spider(self, spider):
"""先建库,再建两张表。"""
database = self.config["database"]
if not re.fullmatch(r"[A-Za-z0-9_]+", database):
raise ValueError(f"非法数据库名:{database}")
self.connection = pymysql.connect(
host=self.config["host"],
port=self.config["port"],
user=self.config["user"],
password=self.config["password"],
charset=self.config["charset"],
autocommit=False,
)
with self.connection.cursor() as cursor:
cursor.execute(
f"CREATE DATABASE IF NOT EXISTS `{database}` "
"DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"
)
self.connection.select_db(database)
cursor.execute(self.stock_ddl)
cursor.execute(self.forex_ddl)
self.connection.commit()
print(f"[MySQL] 已连接 {self.config['host']}:{self.config['port']}/{database}")
def process_item(self, item, spider):
"""按 Item 类型缓存,批量提交。"""
data = ItemAdapter(item)
if isinstance(item, StockItem):
self.stock_rows.append((
data.get("bStockNo"), data.get("bStockName"),
data.get("latestPrice"), data.get("changePercent"),
data.get("changeAmount"), data.get("volume"),
data.get("turnover"), data.get("amplitude"), data.get("high"),
data.get("low"), data.get("openPrice"),
data.get("prevClose"),
))
if len(self.stock_rows) >= 200:
self.flush_stocks()
elif isinstance(item, ForexItem):
self.forex_rows.append((
data.get("currency"), data.get("tbp"), data.get("cbp"),
data.get("tsp"), data.get("csp"),
data.get("quoteDate"), data.get("quoteTime"),
))
return item
def flush_stocks(self) -> None:
"""把股票缓存写入 MySQL。"""
if not self.stock_rows:
return
with self.connection.cursor() as cursor:
cursor.executemany(self.stock_sql, self.stock_rows)
self.connection.commit()
self.stock_count += len(self.stock_rows)
print(f"[MySQL] stocks 已提交 {self.stock_count} 条")
self.stock_rows.clear()
def close_spider(self, spider):
"""收尾提交并输出统计。"""
self.flush_stocks()
if self.forex_rows:
with self.connection.cursor() as cursor:
cursor.executemany(self.forex_sql, self.forex_rows)
self.connection.commit()
self.forex_count = len(self.forex_rows)
print(f"[MySQL] forex 已提交 {self.forex_count} 条")
self.forex_rows.clear()
self.connection.close()
print(f"[MySQL] 完成:stocks={self.stock_count},forex={self.forex_count}")
运行命令:
cd scrapy_assign3
py -X utf8 .\run_stock.py
本次接口返回 5927 条,实际入库也是 5927 条。比第二次作业记录的 5563 条多,原因是行情接口里
新上市股票和接口范围有更新;博客以本次实际运行结果为准。

图 3 MySQL 核验:stocks 共 5927 条,按 id 显示前 8 条
Gitee 作业②:
https://gitee.com/qiu-fuming/2026shujucaijiyuronghejishu/tree/master/作业3
2)心得体会
图片题最重要的是“限制”而不是“抓取”。一开始只想着把所有 <img> 都下载,很容易把站点的 logo、
二维码、广告图也算进去;后来明确按 URL 去重,并且固定 32 页、132 张,结果才可复现。单线程和多线程
的差别也很直观:解析部分很快,真正耗时的是 132 次图片请求,所以线程池对 I/O 密集任务有效,但线程
不是越多越好,8 个线程已经能稳定完成,继续加线程反而容易触发站点限流。
股票题让我第一次把 Item、Pipeline 和 MySQL 串起来。接口里的 f2、f3 这些字段名没有语义,必须
对照行情页面确认;- 这种空值如果直接写进 DECIMAL 会报错,所以要先转成 None。另外,Scrapy 2.20
的入口改成了 async def start(),只写旧版的 start_requests() 不会自动执行,这个问题调试了一会儿
才定位到。批量提交比逐条提交快很多,5000 多条数据写入时差别很明显。
二、作业③:中国银行外汇牌价
1)实验实现
目标页面是中国银行外汇牌价:
https://www.boc.cn/sourcedb/whpj/
页面把牌价放在 id="priceTable" 的表格里,第一行是表头,后面每一行是一个币种。列顺序为:
货币名称、现汇买入价、现钞买入价、现汇卖出价、现钞卖出价、中行折算价、发布日期、发布时间。
作业要求的 TBP/CBP/TSP/CSP/Time 分别对应前四个价格和发布时间。
这一题使用 XPath 定位表格行和单元格:
"""中国银行外汇牌价 Spider。"""
from __future__ import annotations
import scrapy
from scrapy import signals
from demo.items import ForexItem
class ForexSpider(scrapy.Spider):
"""解析中国银行外汇牌价表,Item 交给 MySQL Pipeline。"""
name = "forex"
allowed_domains = ["boc.cn"]
start_urls = ["https://www.boc.cn/sourcedb/whpj/"]
custom_settings = {
"DOWNLOAD_DELAY": 0.2,
"CONCURRENT_REQUESTS": 2,
}
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.parsed = 0
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
spider = super().from_crawler(crawler, *args, **kwargs)
crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
return spider
@staticmethod
def price(value: str):
"""空值、'--' 转 None,防止 DECIMAL 字段报错。"""
try:
return float(value)
except (TypeError, ValueError):
return None
def parse(self, response):
"""按表头顺序提取 8 列,取前 5 个价格和最后两列时间。"""
rows = response.xpath("//table[@id='priceTable']//tr")
self.logger.info("外汇表行数:%s", len(rows))
# 第一行是表头,其余为 45 个币种。
for row in rows[1:]:
cells = [
"".join(cell.xpath(".//text()").getall()).strip()
for cell in row.xpath("./td")
]
if len(cells) < 8 or not cells[0]:
continue
yield ForexItem(
currency=cells[0],
tbp=self.price(cells[1]),
cbp=self.price(cells[2]),
tsp=self.price(cells[3]),
csp=self.price(cells[4]),
quoteDate=cells[6],
quoteTime=cells[7],
)
self.parsed += 1
def spider_closed(self, spider, reason):
"""输出解析行数,方便截图核对。"""
count = self.crawler.stats.get_value("item_scraped_count", 0)
print(f"[完成] 外汇解析={self.parsed},Item 提交={count},原因={reason}")
ForexItem 的字段和 MySQL 表 forex 对应:
| Item 字段 | MySQL 列 | 含义 |
|---|---|---|
currency |
currency |
货币名称 |
tbp |
tbp |
现汇买入价 |
cbp |
cbp |
现钞买入价 |
tsp |
tsp |
现汇卖出价 |
csp |
csp |
现钞卖出价 |
quoteDate / quoteTime |
quoteDate / quoteTime |
发布日期 / 发布时间 |
运行命令:
cd scrapy_assign3
py -X utf8 .\run_forex.py
本次共解析并入库 45 个币种。部分币种(如文莱元)页面只给出部分价格,缺失值在 MySQL 中保存为
NULL,没有用 0 冒充真实价格。

图 4 MySQL 核验:forex 共 45 条,显示前 10 个币种
Gitee 作业③:
https://gitee.com/qiu-fuming/2026shujucaijiyuronghejishu/tree/master/作业3
2)心得体会
外汇题的关键是表格列顺序和空值。页面上有的币种只有现钞价,没有现汇价,如果硬把它转成 0,会让
数据看起来“完整”但实际错误;保存为 NULL 更符合原始页面。XPath 的 tr 选择也要注意表头:先用
//table[@id='priceTable']//tr 取全部行,再在 Python 里跳过第一行,比在 XPath 里写 position()>1
更稳,避免嵌套表格影响位置判断。
浙公网安备 33010602011771号