数据采集第三次作业

作业1

代码及结果

import requests
from bs4 import BeautifulSoup
import os
import time
import threading
import re
from urllib.parse import urljoin


STUDENT_ID = "102402108"

# 取学号后2位控制爬取页数 (尾数 08)
max_pages = int(STUDENT_ID[-2:]) 
if max_pages == 0:
    max_pages = 10  # 防止尾数是00时不翻页

# 取学号后3位控制下载数量 (尾数 108)
max_images = int(STUDENT_ID[-3:])
if max_images == 0:
    max_images = 100 # 防止尾数是000时不下载

print("="*50)
print(f"学号: {STUDENT_ID}")
print(f"规则限制: 最多爬取 {max_pages} 页,最多下载 {max_images} 张图片")
print("="*50)

# ======================================================
# 2. 准备环境
# ======================================================
# 创建 images 文件夹
if not os.path.exists('images'):
    os.makedirs('images')

# 设置请求头,模拟正常浏览器访问,防止被网站反爬拦截
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36'
}

# 全局变量,用于记录已下载图片数量,并防止重复下载
downloaded_count = 0
downloaded_urls = set()
lock = threading.Lock()  # 线程锁,保证多线程下计数的安全


def download_img(img_url):
    global downloaded_count
    

    if downloaded_count >= max_images:
        return
        
    # 检查是否已经下载过该链接
    if img_url in downloaded_urls:
        return

    # 获取图片的后缀名
    ext = os.path.splitext(img_url.split('/')[-1])[1]
    if not ext.lower() in ['.jpg', '.png', '.gif', '.webp', '.bmp']:
        return

    current_count = downloaded_count + 1
    save_path = os.path.join('images', f"{current_count:04d}.jpg") 

    try:
        resp = requests.get(img_url, headers=headers, timeout=10)
        if resp.status_code == 200:
            # 确保下载的是真实图片(大小大于5KB)
            if len(resp.content) > 5000:
                with open(save_path, 'wb') as f:
                    f.write(resp.content)
                
                with lock:
                    downloaded_urls.add(img_url)
                    downloaded_count += 1
                    print(f"[成功] ({downloaded_count}/{max_images}) {save_path}")
    except Exception as e:
        print(f"[失败] 跳过: {e}")

# ======================================================
# 4. 核心爬虫逻辑:获取图片链接并下载
# ======================================================
def start_crawl():
    target_url = "http://www.weather.com.cn"
    
    print(f"\n开始爬取目标网站: {target_url} ...")
    
    # 收集所有页面的图片链接
    all_img_links = []
    
    for i in range(max_pages):
        if i == 0:
            current_page_url = target_url  # 第1页是首页
        else:
            # 气象网翻页通常带有 index.shtml 等特征,这里用 index_{i} 试探
            current_page_url = f"{target_url}/index{i}.shtml" 
        
        print(f"\n正在解析第 {i+1} 页: {current_page_url}")
        try:
            resp = requests.get(current_page_url, headers=headers, timeout=15)
            resp.encoding = 'utf-8'
            soup = BeautifulSoup(resp.text, 'html.parser')
            
            # 提取所有 img 标签
            imgs = soup.find_all('img')
            for img in imgs:
                src = img.get('src') or img.get('data-lazy-src')
                if src:
                    # 拼接成完整的绝对路径
                    full_src = urljoin(current_page_url, src)
                    all_img_links.append(full_src)
        except Exception as e:
            print(f"第 {i+1} 页请求失败,跳过: {e}")
            
    print(f"\n共解析到 {len(all_img_links)} 个图片链接。")

    print("\n--- 开启【单线程】模式开始下载 ---")
    for url in all_img_links:
        if downloaded_count >= max_images:
            break
        download_img(url)
    
    print("\n单线程下载全部完成!\n")

   

if __name__ == "__main__":
    start_crawl()

image

心得

通过这次作业,我进一步掌握了用 Python 写爬虫的基本流程。从发送请求、解析网页到提取数据,每一步都需要耐心调试,也让我对网页结构和数据抓取有了更直观的理解。过程中遇到了数据提取为空、字段格式不统一等问题,通过查阅资料和反复修改代码最终解决了。这次作业让我体会到爬虫的实用性,也为后续学习 Scrapy 框架和数据库存储打下了基础。
作业2
代码及结果

posted @ 2026-11-20 00:42  阿合加依娜尔  阅读(3)  评论(0)    收藏  举报