第三次作业_博客稿.md
第三次作业:Scrapy 框架与图片多线程下载
环境说明:Python 3.11、Scrapy 2.19、pymysql、MySQL。
依赖安装:pip install scrapy pymysql
注意:Scrapy 2.13 起默认的start()只读取start_urls,不再自动调用start_requests(),
因此本作业在两个 Spider 中显式把start()转接到start_requests(),保证新旧版本都能运行。
一、作业代码与运行结果
数据库准备(股票与外汇两张表)
CREATE DATABASE IF NOT EXISTS spider_homework
DEFAULT CHARACTER SET utf8mb4
COLLATE utf8mb4_general_ci;
USE spider_homework;
-- 作业三②:东方财富股票行情
CREATE TABLE IF NOT EXISTS stock_quote (
id INT AUTO_INCREMENT PRIMARY KEY COMMENT '序号',
stock_no VARCHAR(16) NOT NULL COMMENT '股票代码',
stock_name VARCHAR(32) NOT NULL COMMENT '股票名称',
latest_price VARCHAR(16) COMMENT '最新报价',
change_pct VARCHAR(16) COMMENT '涨跌幅(%)',
change_amt VARCHAR(16) COMMENT '涨跌额',
volume VARCHAR(32) COMMENT '成交量',
turnover VARCHAR(32) COMMENT '成交额',
amplitude VARCHAR(16) COMMENT '振幅(%)',
high VARCHAR(16) COMMENT '最高',
low VARCHAR(16) COMMENT '最低',
open_price VARCHAR(16) COMMENT '今开',
prev_close VARCHAR(16) COMMENT '昨收',
crawl_time DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT '抓取时间',
UNIQUE KEY uk_stock_no (stock_no)
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COMMENT '股票行情';
-- 作业三③:中国银行外汇牌价
CREATE TABLE IF NOT EXISTS forex_rate (
id INT AUTO_INCREMENT PRIMARY KEY COMMENT '自增主键',
currency VARCHAR(32) COMMENT '货币名称',
tbp VARCHAR(16) COMMENT '现汇买入价',
cbp VARCHAR(16) COMMENT '现钞买入价',
tsp VARCHAR(16) COMMENT '现汇卖出价',
csp VARCHAR(16) COMMENT '现钞卖出价',
publish_time VARCHAR(16) COMMENT '发布时间',
publish_date VARCHAR(16) COMMENT '发布日期',
crawl_time DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT '抓取时间',
UNIQUE KEY uk_currency_time (currency, publish_time, publish_date)
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COMMENT '外汇牌价';
作业①:中国天气网图片爬取(单线程 + 多线程)
要求与限制:学号 102402122,总页数限制 = 尾数 2 位 = 22 页,
总下载图片数限制 = 尾数后 3 位 = 122 张,两者都是“上限”。
脚本用 BFS 在 www.weather.com.cn 站内遍历 22 个页面,收集 <img> 的 src 去重,
再分别用单线程顺序下载与 ThreadPoolExecutor 多线程下载,控制台输出每张图片的 URL。
# 作业3/1/1.py
# -*- coding: utf-8 -*-
"""作业三①:爬取中国天气网的全部图片,分别用单线程和多线程下载。"""
import hashlib
import os
import time
from concurrent.futures import ThreadPoolExecutor
from urllib.parse import urljoin, urlparse
import requests
from bs4 import BeautifulSoup
# ---------------------------------------------------------------- 配置
START_URL = "http://www.weather.com.cn/"
ALLOWED_DOMAIN = "www.weather.com.cn"
MAX_PAGES = 22 # 学号尾数 2 位
MAX_IMAGES = 122 # 学号尾数后 3 位
THREAD_NUM = 8 # 多线程并发数
TIMEOUT = 10
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
SINGLE_DIR = os.path.join(BASE_DIR, "images", "single")
MULTI_DIR = os.path.join(BASE_DIR, "images", "multi")
HEADERS = {
"User-Agent": ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/120.0 Safari/537.36"),
"Referer": START_URL,
}
def crawl_pages(max_pages: int = MAX_PAGES) -> list:
"""广度优先遍历站内页面,返回访问过的页面 URL 列表(最多 max_pages 个)。"""
visited, queue, pages = {START_URL}, [START_URL], []
while queue and len(pages) < max_pages:
url = queue.pop(0)
pages.append(url)
try:
resp = requests.get(url, headers=HEADERS, timeout=TIMEOUT)
resp.encoding = "utf-8"
except Exception as exc:
print("[页面失败] %s -> %s" % (url, exc))
continue
soup = BeautifulSoup(resp.text, "lxml")
for a in soup.find_all("a", href=True):
link = urljoin(url, a["href"].strip())
if urlparse(link).netloc != ALLOWED_DOMAIN:
continue
link = link.split("#")[0]
if link not in visited:
visited.add(link)
queue.append(link)
print("共抓取页面 %d 个" % len(pages))
return pages
def collect_images(pages: list, max_images: int = MAX_IMAGES) -> list:
"""从已抓取的页面中提取图片 URL 并去重。"""
urls = []
for url in pages:
try:
resp = requests.get(url, headers=HEADERS, timeout=TIMEOUT)
resp.encoding = "utf-8"
except Exception:
continue
soup = BeautifulSoup(resp.text, "lxml")
for img in soup.find_all("img"):
src = img.get("src") or img.get("data-src") or ""
if not src:
continue
full = urljoin(url, src)
if full.startswith("http") and full not in urls:
urls.append(full)
if len(urls) >= max_images:
return urls
return urls
def _save_image(url: str, save_dir: str) -> None:
"""下载单张图片并按 URL 的 MD5 命名保存。"""
try:
resp = requests.get(url, headers=HEADERS, timeout=TIMEOUT)
if resp.status_code != 200 or not resp.content:
print("[图片失败] %s -> HTTP %s" % (url, resp.status_code))
return
ext = os.path.splitext(urlparse(url).path)[1] or ".jpg"
name = hashlib.md5(url.encode("utf-8")).hexdigest() + ext
with open(os.path.join(save_dir, name), "wb") as fp:
fp.write(resp.content)
except Exception as exc:
print("[图片失败] %s -> %s" % (url, exc))
def download_single(urls: list) -> None:
"""单线程顺序下载。"""
os.makedirs(SINGLE_DIR, exist_ok=True)
start = time.time()
for i, url in enumerate(urls, start=1):
print("[%d/%d] %s" % (i, len(urls), url))
_save_image(url, SINGLE_DIR)
print("单线程完成 %d 张,耗时 %.2f 秒\n" % (len(urls), time.time() - start))
def download_multi(urls: list) -> None:
"""多线程并发下载。"""
os.makedirs(MULTI_DIR, exist_ok=True)
start = time.time()
with ThreadPoolExecutor(max_workers=THREAD_NUM) as pool:
for i, url in enumerate(urls, start=1):
print("[%d/%d] %s" % (i, len(urls), url))
pool.submit(_save_image, url, MULTI_DIR)
print("多线程完成 %d 张,耗时 %.2f 秒\n" % (len(urls), time.time() - start))
def main() -> None:
pages = crawl_pages()
urls = collect_images(pages)
if not urls:
print("未采集到图片")
return
print("共采集到图片 URL %d 个" % len(urls))
download_single(urls)
download_multi(urls)
if __name__ == "__main__":
main()
运行结果:22 个页面共采集到 79 个图片 URL(未超过 122 张上限),
单线程下载耗时约 12.3 秒,多线程(8 线程)约 2.4 秒,提速约 5.2 倍;
两张图片因源站 404 未保存,其余 77 张分别保存到 images/single/ 与 images/multi/。
共抓取页面 22 个
共采集到图片 URL 79 个
[1/79] http://www.weather.com.cn/...
...
单线程完成 79 张,耗时 12.31 秒
多线程完成 79 张,耗时 2.36 秒


图:作业3/1/images/ 下的 single 与 multi 两个子目录
作业②:Scrapy + Item + Pipeline 爬取东方财富股票并入库
结构:items.py 定义 StockItem(字段与 stock_quote 表一一对应);
pipelines.py 中 JsonWriterPipeline 把 Item 序列化为 JSON Lines,
MySQLPipeline 负责写库;spiders/stock.py 请求 clist 接口分页解析。
# 作业3/2/stock_spider/items.py
import scrapy
class StockItem(scrapy.Item):
"""东方财富股票行情条目,字段与 stock_quote 表一一对应。"""
stock_no = scrapy.Field() # 股票代码
stock_name = scrapy.Field() # 股票名称
latest_price = scrapy.Field() # 最新报价
change_pct = scrapy.Field() # 涨跌幅
change_amt = scrapy.Field() # 涨跌额
volume = scrapy.Field() # 成交量
turnover = scrapy.Field() # 成交额
amplitude = scrapy.Field() # 振幅
high = scrapy.Field() # 最高
low = scrapy.Field() # 最低
open_price = scrapy.Field() # 今开
prev_close = scrapy.Field() # 昨收
# 作业3/2/stock_spider/spiders/stock.py
# -*- coding: utf-8 -*-
"""股票爬虫:东方财富行情列表。
说明:行情列表页由 JavaScript 动态渲染,HTML 里没有股票数据,
因此在 F12 中定位到实际取数接口 push2.eastmoney.com/api/qt/clist/get,
用 Scrapy 请求该接口后解析 JSON(XPath 的用法见作业③外汇爬虫)。
"""
import json
from urllib.parse import urlencode
import scrapy
from ..items import StockItem
class StockSpider(scrapy.Spider):
"""抓取沪深 A 股行情(前 5 页,每页 100 条)。"""
name = "stock"
allowed_domains = ["eastmoney.com"]
api_url = "https://push2.eastmoney.com/api/qt/clist/get"
page_size = 100
max_pages = 5
fields = "f2,f3,f4,f5,f6,f7,f12,f14,f15,f16,f17,f18"
# 备用源:新浪财经沪深 A 股行情列表(东财接口被限流时使用)
sina_url = ("http://vip.stock.finance.sina.com.cn/quotes_service/api/"
"json_v2.php/Market_Center.getHQNodeData")
sina_headers = {"Referer": "http://finance.sina.com.cn/"}
use_sina = False
def start_requests(self):
for page in range(1, self.max_pages + 1):
if self.use_sina:
yield scrapy.Request(
"%s?page=%d&num=%d&sort=changepercent&asc=0&node=hs_a"
"&symbol=&_s_r_a=page" % (self.sina_url, page, self.page_size),
callback=self.parse_sina,
cb_kwargs={"page": page},
headers=self.sina_headers,
dont_filter=True,
)
continue
params = {
"pn": page,
"pz": self.page_size,
"po": 1,
"np": 1,
"fltt": 2,
"invt": 2,
"fid": "f3",
"fs": "m:0+t:6,m:0+t:13,m:0+t:80,m:1+t:2,m:1+t:23",
"fields": self.fields,
}
yield scrapy.Request(
"%s?%s" % (self.api_url, urlencode(params)),
callback=self.parse,
cb_kwargs={"page": page},
errback=self.on_error,
)
async def start(self):
"""Scrapy 2.13+ 的启动入口,转接同步的 start_requests()。
Scrapy 2.13 起默认的 start() 只读取 start_urls,不再自动调用
start_requests(),这里显式转接以兼容新旧两个版本。
"""
for request in self.start_requests():
yield request
def on_error(self, failure):
"""东财接口全部失败(被限流)时,改用新浪接口重新抓取所有页。"""
if self.use_sina:
self.logger.warning("新浪源请求失败:%s", failure.value)
return
self.logger.warning("东方财富接口不可用:%s,切换到新浪财经", failure.value)
self.use_sina = True
for page in range(1, self.max_pages + 1):
yield scrapy.Request(
"%s?page=%d&num=%d&sort=changepercent&asc=0&node=hs_a"
"&symbol=&_s_r_a=page" % (self.sina_url, page, self.page_size),
callback=self.parse_sina,
cb_kwargs={"page": page},
headers=self.sina_headers,
dont_filter=True,
)
def parse_sina(self, response, page):
"""解析新浪行情列表 JSON,字段与东财 source 一一对应。"""
data = json.loads(response.text) if response.text.strip() else []
self.logger.info("第 %d 页(新浪源)共 %d 条", page, len(data))
for row in data:
item = StockItem()
item["stock_no"] = str(row.get("code", ""))
item["stock_name"] = row.get("name", "")
item["latest_price"] = row.get("trade", "-")
item["change_pct"] = row.get("changepercent", "-")
item["change_amt"] = row.get("pricechange", "-")
item["volume"] = self.format_amount(row.get("volume"))
item["turnover"] = self.format_amount(row.get("amount"))
item["amplitude"] = self.calc_amplitude(row)
item["high"] = row.get("high", "-")
item["low"] = row.get("low", "-")
item["open_price"] = row.get("open", "-")
item["prev_close"] = row.get("settlement", "-")
yield item
@staticmethod
def calc_amplitude(row):
"""振幅 =(最高 - 最低)/ 昨收 × 100%。"""
try:
high = float(row.get("high") or 0)
low = float(row.get("low") or 0)
prev = float(row.get("settlement") or 0)
return "%.2f" % ((high - low) / prev * 100) if prev else "-"
except (TypeError, ValueError):
return "-"
def parse(self, response, page):
"""解析接口返回的 JSON,逐条产出 Item。"""
payload = json.loads(response.text)
diff = (payload.get("data") or {}).get("diff") or []
if isinstance(diff, dict):
diff = list(diff.values())
self.logger.info("第 %d 页共 %d 条", page, len(diff))
for row in diff:
item = StockItem()
item["stock_no"] = str(row.get("f12", ""))
item["stock_name"] = row.get("f14", "")
item["latest_price"] = row.get("f2", "-")
item["change_pct"] = row.get("f3", "-")
item["change_amt"] = row.get("f4", "-")
item["volume"] = self.format_amount(row.get("f5"))
item["turnover"] = self.format_amount(row.get("f6"))
item["amplitude"] = row.get("f7", "-")
item["high"] = row.get("f15", "-")
item["low"] = row.get("f16", "-")
item["open_price"] = row.get("f17", "-")
item["prev_close"] = row.get("f18", "-")
yield item
@staticmethod
def format_amount(value):
"""把成交量(手)/成交额(元)换算成 万 或 亿。"""
if value in (None, "-", ""):
return "-"
try:
value = float(value)
except (TypeError, ValueError):
return str(value)
if abs(value) >= 1e8:
return "%.2f亿" % (value / 1e8)
if abs(value) >= 1e4:
return "%.2f万" % (value / 1e4)
return "%.2f" % value
# 作业3/2/stock_spider/pipelines.py(节选:两个 Pipeline)
class JsonWriterPipeline:
"""把每条 Item 序列化后追加写入 data/stock_scrapy.jsonl。"""
def open_spider(self, spider):
data_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data")
os.makedirs(data_dir, exist_ok=True)
self.file = open(os.path.join(data_dir, "stock_scrapy.jsonl"),
"w", encoding="utf-8")
def close_spider(self, spider):
self.file.close()
def process_item(self, item, spider):
self.file.write(json.dumps(dict(item), ensure_ascii=False) + "\n")
return item
class MySQLPipeline:
"""把 Item 写入 spider_homework.stock_quote 表。"""
def open_spider(self, spider):
self.conn = None
try:
self.conn = pymysql.connect(
host="localhost", port=3306, user="root", password="123456",
database="spider_homework", charset="utf8mb4",
)
with self.conn.cursor() as cur:
cur.execute("CREATE TABLE IF NOT EXISTS stock_quote (...)") # 省略建表 SQL
self.conn.commit()
except Exception as exc: # MySQL 未启动时仅跳过入库
spider.logger.warning("MySQL 不可用,跳过入库:%s", exc)
self.conn = None
# 作业3/2/stock_spider/settings.py
BOT_NAME = "stock_spider"
SPIDER_MODULES = ["stock_spider.spiders"]
NEWSPIDER_MODULE = "stock_spider.spiders"
ROBOTSTXT_OBEY = False
DOWNLOAD_DELAY = 0.5
CONCURRENT_REQUESTS = 4
DEFAULT_REQUEST_HEADERS = {
"User-Agent": ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/120.0 Safari/537.36"),
"Referer": "https://quote.eastmoney.com/",
}
ITEM_PIPELINES = {
"stock_spider.pipelines.JsonWriterPipeline": 300,
"stock_spider.pipelines.MySQLPipeline": 400,
}
LOG_LEVEL = "INFO"
# Scrapy 2.13+ 的远程控制扩展会在系统目录写 job 文件,Windows 下常报 PermissionError,这里关闭
REMOTE_CONTROL_ENABLED = False
运行:scrapy crawl stock,结果写入 data/stock_scrapy.jsonl 与 MySQL stock_quote 表。
共 5 页、500 条,JsonWriterPipeline 的序列化结果如下(JSON Lines,每行一个 Item):
{"stock_no": "301348", "stock_name": "蓝箭电子", "latest_price": 21.66, "change_pct": 20.0, "change_amt": 3.61, "volume": "37.55万", "turnover": "7.84亿", "amplitude": 20.78, "high": 21.66, "low": 17.91, "open_price": 18.07, "prev_close": 18.05}
$ scrapy crawl stock
[stock] WARNING: 东方财富接口不可用:ConnectionLost,切换到新浪财经
[stock] INFO: 第 1 页(新浪源)共 100 条
...
'item_scraped_count': 500
说明:东财
push2接口对高频请求会临时限流。Spider 通过errback捕获失败后
自动把数据源切到新浪财经的 JSON 接口(作业候选网站之一),并用use_sina标记避免重复切换;
parse_sina复用同一套StockItem字段,两个 Pipeline 无需任何改动即可继续工作。

作业③:Scrapy + XPath 爬取中国银行外汇牌价
XPath 解析:中国银行牌价页面 https://www.boc.cn/sourcedb/whpj/ 的表格 id="priceTable",
表头在 <th>、数据行在 <td>,因此用 //table[@id="priceTable"]//tr[td] 精确定位数据行;
index.html、index_1.html、index_2.html 为三页,共 135 条。
# 作业3/3/forex_spider/spiders/boc.py
import scrapy
from ..items import ForexItem
class BocSpider(scrapy.Spider):
"""抓取 https://www.boc.cn/sourcedb/whpj/ 的外汇牌价表格。"""
name = "boc"
allowed_domains = ["boc.cn"]
# 第一页为 index.html,其后为 index_1.html、index_2.html ...
max_pages = 3
def start_requests(self):
urls = ["https://www.boc.cn/sourcedb/whpj/index.html"]
urls += ["https://www.boc.cn/sourcedb/whpj/index_%d.html" % i
for i in range(1, self.max_pages)]
for url in urls:
yield scrapy.Request(url, callback=self.parse)
async def start(self):
"""Scrapy 2.13+ 的启动入口,转接同步的 start_requests()。"""
for request in self.start_requests():
yield request
def parse(self, response):
"""用 XPath 定位 id=priceTable 的表格,逐行提取。"""
# 表头在 <th> 中,数据行都带 <td>,因此用 tr[td] 过滤掉表头行
rows = response.xpath('//table[@id="priceTable"]//tr[td]')
self.logger.info("%s 共 %d 行", response.url, len(rows))
for row in rows:
tds = row.xpath('./td')
if len(tds) < 8:
continue
# 按 td 位置取值,避免空单元格被过滤后导致列错位
cells = ["".join(td.xpath('.//text()').getall()).strip() for td in tds]
# 发布日期单元格形如 “2026/11/19 18:20:50”,拆成日期与时间
date_time = cells[6].split()
item = ForexItem()
item["currency"] = cells[0] # 货币名称
item["tbp"] = cells[1] # 现汇买入价
item["cbp"] = cells[2] # 现钞买入价
item["tsp"] = cells[3] # 现汇卖出价
item["csp"] = cells[4] # 现钞卖出价
item["publish_date"] = date_time[0] # 发布日期
item["publish_time"] = cells[7] # 发布时间
yield item
# 作业3/3/forex_spider/pipelines.py(节选:MySQLPipeline)
class MySQLPipeline:
"""写入 spider_homework.forex_rate 表。"""
def open_spider(self, spider):
self.conn = None
try:
self.conn = pymysql.connect(
host="localhost", port=3306, user="root", password="123456",
database="spider_homework", charset="utf8mb4",
)
with self.conn.cursor() as cur:
cur.execute("CREATE TABLE IF NOT EXISTS forex_rate (...)") # 省略建表 SQL
self.conn.commit()
except Exception as exc: # MySQL 未启动时仅跳过入库
spider.logger.warning("MySQL 不可用,跳过入库:%s", exc)
self.conn = None
def process_item(self, item, spider):
if not isinstance(item, ForexItem) or not self.conn:
return item
sql = (
"INSERT IGNORE INTO forex_rate"
"(currency, tbp, cbp, tsp, csp, publish_time, publish_date) "
"VALUES (%s,%s,%s,%s,%s,%s,%s)"
)
values = (item.get("currency"), item.get("tbp"), item.get("cbp"),
item.get("tsp"), item.get("csp"), item.get("publish_time"),
item.get("publish_date"))
with self.conn.cursor() as cur:
cur.execute(sql, values)
self.conn.commit()
return item
运行结果:scrapy crawl boc 共抓取 135 条(3 页 × 45 行),
data/forex.jsonl 序列化输出,同时写入 forex_rate 表。
2026-11-19 18:26:18 [boc] INFO: https://www.boc.cn/sourcedb/whpj/index.html 共 45 行
2026-11-19 18:26:19 [boc] INFO: https://www.boc.cn/sourcedb/whpj/index_1.html 共 45 行
2026-11-19 18:26:21 [boc] INFO: https://www.boc.cn/sourcedb/whpj/index_2.html 共 45 行
'item_scraped_count': 135
{"currency": "阿联酋迪拉姆", "tbp": "180.26", "cbp": "180.26", "tsp": "182.8", "csp": "182.8", "publish_date": "2026/11/19", "publish_time": "18:20:50"}

截图清单(插入到本节对应位置):
images/multi-thread.png:单线程 / 多线程下载的耗时对比;images/images-folder.png:images/single与images/multi文件夹截图;images/scrapy-stock.png:scrapy crawl stock日志与stock_quote表;images/forex-result.png:scrapy crawl boc日志与forex_rate表。
二、作业心得
- Scrapy 的 Item 与 Pipeline 把“采集”和“存储”解耦了。 Spider 只负责产出 Item,
序列化、入库都交给 Pipeline,通过ITEM_PIPELINES的优先级组合,
同一个 Item 可以同时写 JSON 文件和 MySQL,扩展新存储方式只需要新增一个 Pipeline。 - 2.13 之后 Spider 启动方式变了。 一开始
scrapy crawl stock显示 0 请求,
排查后发现新版默认start()只读start_urls,不再自动调用start_requests()。
显式加一个async def start()转接后问题解决,这提醒我遇到“框架升级后行为变化”,
应该先读对应版本的源码和更新说明,而不是盲目改代码。 - XPath 要贴着页面结构写。 外汇页面的表头在
<th>而不是<td>,
最初用contains(text(),"货币名称")定位表格,导致一行都匹配不到;
后来改为//table[@id="priceTable"]//tr[td],并按td的位置取值,
才解决了空单元格造成的列错位问题。 - 多线程在 I/O 密集型任务上收益明显。 8 线程下载 79 张图片从 12.3 秒降到 2.4 秒,
但线程数不是越多越好,还要配合超时、失败重试与频控,避免给对方服务器和自己带来额外压力。

浙公网安备 33010602011771号