数据采集第三次作业
作业1
代码及结果
import requests
from bs4 import BeautifulSoup
import os
import time
import threading
import re
from urllib.parse import urljoin
STUDENT_ID = "102402108"
# 取学号后2位控制爬取页数 (尾数 08)
max_pages = int(STUDENT_ID[-2:])
if max_pages == 0:
max_pages = 10 # 防止尾数是00时不翻页
# 取学号后3位控制下载数量 (尾数 108)
max_images = int(STUDENT_ID[-3:])
if max_images == 0:
max_images = 100 # 防止尾数是000时不下载
print("="*50)
print(f"学号: {STUDENT_ID}")
print(f"规则限制: 最多爬取 {max_pages} 页,最多下载 {max_images} 张图片")
print("="*50)
# ======================================================
# 2. 准备环境
# ======================================================
# 创建 images 文件夹
if not os.path.exists('images'):
os.makedirs('images')
# 设置请求头,模拟正常浏览器访问,防止被网站反爬拦截
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36'
}
# 全局变量,用于记录已下载图片数量,并防止重复下载
downloaded_count = 0
downloaded_urls = set()
lock = threading.Lock() # 线程锁,保证多线程下计数的安全
def download_img(img_url):
global downloaded_count
if downloaded_count >= max_images:
return
# 检查是否已经下载过该链接
if img_url in downloaded_urls:
return
# 获取图片的后缀名
ext = os.path.splitext(img_url.split('/')[-1])[1]
if not ext.lower() in ['.jpg', '.png', '.gif', '.webp', '.bmp']:
return
current_count = downloaded_count + 1
save_path = os.path.join('images', f"{current_count:04d}.jpg")
try:
resp = requests.get(img_url, headers=headers, timeout=10)
if resp.status_code == 200:
# 确保下载的是真实图片(大小大于5KB)
if len(resp.content) > 5000:
with open(save_path, 'wb') as f:
f.write(resp.content)
with lock:
downloaded_urls.add(img_url)
downloaded_count += 1
print(f"[成功] ({downloaded_count}/{max_images}) {save_path}")
except Exception as e:
print(f"[失败] 跳过: {e}")
# ======================================================
# 4. 核心爬虫逻辑:获取图片链接并下载
# ======================================================
def start_crawl():
target_url = "http://www.weather.com.cn"
print(f"\n开始爬取目标网站: {target_url} ...")
# 收集所有页面的图片链接
all_img_links = []
for i in range(max_pages):
if i == 0:
current_page_url = target_url # 第1页是首页
else:
# 气象网翻页通常带有 index.shtml 等特征,这里用 index_{i} 试探
current_page_url = f"{target_url}/index{i}.shtml"
print(f"\n正在解析第 {i+1} 页: {current_page_url}")
try:
resp = requests.get(current_page_url, headers=headers, timeout=15)
resp.encoding = 'utf-8'
soup = BeautifulSoup(resp.text, 'html.parser')
# 提取所有 img 标签
imgs = soup.find_all('img')
for img in imgs:
src = img.get('src') or img.get('data-lazy-src')
if src:
# 拼接成完整的绝对路径
full_src = urljoin(current_page_url, src)
all_img_links.append(full_src)
except Exception as e:
print(f"第 {i+1} 页请求失败,跳过: {e}")
print(f"\n共解析到 {len(all_img_links)} 个图片链接。")
print("\n--- 开启【单线程】模式开始下载 ---")
for url in all_img_links:
if downloaded_count >= max_images:
break
download_img(url)
print("\n单线程下载全部完成!\n")
if __name__ == "__main__":
start_crawl()

心得
通过这次作业,我进一步掌握了用 Python 写爬虫的基本流程。从发送请求、解析网页到提取数据,每一步都需要耐心调试,也让我对网页结构和数据抓取有了更直观的理解。过程中遇到了数据提取为空、字段格式不统一等问题,通过查阅资料和反复修改代码最终解决了。这次作业让我体会到爬虫的实用性,也为后续学习 Scrapy 框架和数据库存储打下了基础。
作业2
代码及结果
浙公网安备 33010602011771号