数据采集实践第一次作业

作业①

1）、用requests和BeautifulSoup库方法定向爬取给定网址（http://www.shanghairanking.cn/rankings/bcur/2020 ）的数据，屏幕打印爬取的大学排名信息。

代码

from urllib.request import urlopen, Request
from bs4 import BeautifulSoup
import re
from tabulate import tabulate

url = "http://www.shanghairanking.cn/rankings/bcur/2020"
N = 44

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) \
        Chrome/120.0.0.0 Safari/537.36"
}
req = Request(url, headers=headers)
response = urlopen(req)
html = response.read().decode("utf-8")

soup = BeautifulSoup(html, "html.parser")

rows = soup.select(".rk-table tbody tr")

# 取前N行
data_list = []
for row in rows[:N]:
    tds = row.find_all("td")
    rank = tds[0].get_text(strip=True)
    name_ = tds[1].get_text(strip=True)
    name = re.match(r'^[\u4e00-\u9fa5]+', name_).group()
    province = tds[2].get_text(strip=True)
    school_type = tds[3].get_text(strip=True)
    score = tds[4].get_text(strip=True)
    data_list.append([rank,name,province,school_type,score])
print(tabulate(data_list, headers=["排名","学校名称","省市","学校类型","总分"]))

运行结果

2）、心得体会

掌握了利用网页html定位需要爬取的内容的能力
学会用tabulate输出格式化中文内容

作业②

1）、用requests和re库方法设计某个商城（自已选择）商品比价定向爬虫，爬取该商城，以关键词“书包”搜索页面的数据，爬取商品名称和价格。

代码

from bs4 import BeautifulSoup
import db
from dotenv import load_dotenv
import os
import re
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
import time

# 加载环境变量
parent_dir = os.path.dirname(__file__)
load_dotenv(os.path.join(parent_dir, ".env"))
UA = os.getenv("USER_AGENT")
COOKIE = os.getenv("COOKIE")

# Chrome浏览器配置
chrome_options = Options()
chrome_options.add_argument(f"user-agent={UA}")
# 可选：无头模式（不弹出浏览器窗口，后台运行）
# chrome_options.add_argument("--headless=new")
chrome_options.add_argument("--disable-blink-features=AutomationControlled")
chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
chrome_options.add_experimental_option("useAutomationExtension", False)

# 启动浏览器
driver = webdriver.Chrome(options=chrome_options)
driver.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")

page = 1
url1 = "https://s.taobao.com/search?clientPreloadId=preload_1791097366716&commend=all&ie=utf8&initiative_id=tbindexz_20170306&page="
url2 = "&preLoadOrigin=https%3A%2F%2Fwww.taobao.com&q=%E4%B9%A6%E5%8C%85&search_type=item&sourceId=tb.index&spm=a21bo.jianhua%2Fa.search_manual.0&ssid=s5-e&tab=all"
url = url1 + str(page) + url2

db.init_db()
db.clear_all()

try:
    driver.get(url)
    # 等待页面加载+JS渲染，根据网速调整，淘宝比较慢建议3~5秒
    time.sleep(4)

    # 把你的淘宝Cookie字符串注入浏览器
    if COOKIE:
        # 先访问域名才能添加cookie
        driver.get("https://s.taobao.com")
        time.sleep(1)
        cookie_list = COOKIE.split("; ")
        for ck in cookie_list:
            if "=" in ck:
                k, v = ck.split("=",1)
                driver.add_cookie({"name":k, "value":v, "domain":".taobao.com"})
        # 重新打开目标页面
        driver.get(url)
        time.sleep(4)

    # 获取渲染完成后的页面源码
    page_html = driver.page_source
    with open(os.path.join(parent_dir, "test.html"), "w", encoding="utf-8") as f:
        f.write(page_html)
    print("已保存渲染后的页面 test.html")

    soup = BeautifulSoup(page_html, "lxml")
    tag_list = soup.find_all("div",class_=re.compile(r"^tbpc-row"))
    print(f"找到 tbpc-row 的数量：{len(tag_list)}")
    if len(tag_list) == 0:
        print("⚠️ 警告：没找到商品容器，触发滑块/登录验证")
    else:
        outer_tag = tag_list[0]
        # 直接找商品卡片，class以 doubleCard-- 开头
        lis = outer_tag.find_all("div", class_=re.compile(r"^doubleCard--"))
        print(f"本次抓到商品条数{len(lis)}")
        for tag_item in lis:
            try:
                tag = tag_item.find("div",class_=re.compile(r"^priceWrapper"))
                priceInt = tag.find("div",class_=re.compile(r"^priceInt")).text
                priceFloat = tag.find("div",class_=re.compile(r"^priceFloat")).text
                name = tag_item.find("div",class_=re.compile(r"^title")).get("title","无商品属性")
                db.insert_row(name,priceInt+priceFloat)
            except Exception as err:
                print(f"单个商品解析失败：{err}")
        db.get_all_and_print()

except Exception as err:
    print(f"整体异常：{err}")
finally:
    # 无论是否报错，关闭浏览器
    driver.quit()

db

import sqlite3
from tabulate import tabulate
# 数据库文件名称
DB_FILE = "goods.db"

def init_db():
    """初始化数据库，创建商品表（商品名字，商品价格）"""
    conn = sqlite3.connect(DB_FILE)
    cur = conn.cursor()
    # 建表，goods_name商品名，goods_price商品价格
    cur.execute('''
    CREATE TABLE IF NOT EXISTS goods(
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        goods_name TEXT,
        goods_price TEXT
    )
    ''')
    conn.commit()
    conn.close()
    print("数据库初始化完成")

def insert_row(goods_name: str, goods_price: str):
    """逐行插入一条商品数据"""
    conn = sqlite3.connect(DB_FILE)
    cur = conn.cursor()
    cur.execute("INSERT INTO goods(goods_name, goods_price) VALUES (?, ?)", (goods_name, goods_price))
    conn.commit()
    conn.close()
    # print(f"插入成功：{goods_name} | {goods_price}")

def get_one_row(row_id: int):
    """读取特定行数据，按id查询"""
    conn = sqlite3.connect(DB_FILE)
    cur = conn.cursor()
    cur.execute("SELECT * FROM goods WHERE id=?", (row_id,))
    res = cur.fetchone()
    conn.close()
    return res

def get_all_and_print():
    """读取全部数据并打印，带表头"""
    conn = sqlite3.connect(DB_FILE)
    cur = conn.cursor()
    cur.execute("SELECT * FROM goods")
    all_data = cur.fetchall()
    conn.close()
    # 固定宽度格式化
    # print(f"{'id':<4}{'商品名字':<70}{'商品价格':<8}")
    data_list = []
    print("-" * 90)
    for row in all_data:
        data_list.append([row[0],float(row[2]),row[1]])
        # print(f"{row[0]:<4}{row[1]:<70}{row[2]:<8}")
    print(tabulate(data_list,headers=["id","商品价格","商品名字"]))
    return all_data


def clear_all():
    """一键清空所有数据，保留表结构"""
    conn = sqlite3.connect(DB_FILE)
    cur = conn.cursor()
    cur.execute("DELETE FROM goods")
    # 重置自增id
    cur.execute("UPDATE sqlite_sequence SET seq=0 WHERE name='goods'")
    conn.commit()
    conn.close()
    print("✅所有商品数据已清空")

运行结果

2）、心得体会

电商网页反爬能力很强，改用webdriver库使用cookie注入的方法绕过反爬
使用数据库来储存爬取到的数据

作业③

1）、爬取一个给定网页（ https://news.fzu.edu.cn/yxfd.htm ）或者自选网页的所有JPEG、JPG或PNG格式图片文件

代码

from urllib.request import urlopen, Request
from urllib.parse import urljoin
from bs4 import BeautifulSoup
import re
import os

# ========= 配置 =========
target_url = "https://news.fzu.edu.cn/yxfd.htm"
save_dir = os.path.join(os.path.dirname(__file__),"images_save")
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
}
# 正则：匹配jpg jpeg png，忽略大小写
img_pattern = re.compile(r"\.(jpg|jpeg|png)$", re.IGNORECASE)
# =========================

# 创建保存文件夹
os.makedirs(save_dir, exist_ok=True)

# 请求网页
req = Request(target_url, headers=headers)
response = urlopen(req)
html = response.read().decode("utf-8")
soup = BeautifulSoup(html, "lxml")

# 获取所有img标签
img_tags = soup.find_all("img")
img_url_set = set() # 用集合去重，防止同一张图片多次抓取

for img in img_tags:
    src = img.get("src")
    if not src:
        continue
    # 转绝对链接
    full_url = urljoin(target_url, src)
    # 过滤后缀
    if img_pattern.search(full_url):
        img_url_set.add(full_url)

# 开始下载
print(f"找到 {len(img_url_set)} 张jpg/png图片，开始下载")
for idx, img_url in enumerate(img_url_set):
    try:
        req_img = Request(img_url, headers=headers)
        img_data = urlopen(req_img).read()
        # 提取文件名
        filename = os.path.basename(img_url).split("?")[0] # 去掉url参数
        save_path = os.path.join(save_dir, filename)
        with open(save_path, "wb") as f:
            f.write(img_data)
        print(f"✅ 已下载: {filename}")
    except Exception as e:
        print(f"❌ 下载失败 {img_url} , 错误: {e}")

print("全部任务结束！图片保存在", save_dir)

运行结果

2）、心得体会

掌握了定向爬取并下载图片保存到本地的能力