通用爬虫

python

import requests
import time
import random
import csv
import os
from bs4 import BeautifulSoup
from urllib.parse import urljoin
import logging
from fake_useragent import UserAgent
import pandas as pd

配置日志

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(name)

class GenericSpider:
def init(self):
# 初始化UserAgent随机生成器
self.ua = UserAgent()
self.session = requests.Session()

设置请求头

self.headers = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,/;q=0.8',
'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
'Accept-Encoding': 'gzip, deflate',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1',
}

数据存储路径

self.data_dir = 'crawled_data'
if not os.path.exists(self.data_dir):
os.makedirs(self.data_dir)

def get_random_headers(self):
"""生成随机请求头"""
headers = self.headers.copy()
headers['User-Agent'] = self.ua.random
return headers

def fetch_page(self, url, retry_times=3):
"""获取页面内容，支持重试"""
for i in range(retry_times):
try:
# 随机延时，避免请求过快
time.sleep(random.uniform(1, 3))

发送请求

response = self.session.get(
url,
headers=self.get_random_headers(),
timeout=10
)

检查响应状态

if response.status_code == 200:
# 检查编码
if response.encoding:
response.encoding = response.apparent_encoding
return response.text
elif response.status_code == 403:
logger.warning(f"被禁止访问 {url}，尝试更换IP或增加延时")
time.sleep(random.uniform(5, 10))
elif response.status_code == 404:
logger.error(f"页面不存在 {url}")
return None
else:
logger.warning(f"状态码 {response.status_code}，重试 {i+1}/{retry_times}")

except requests.exceptions.RequestException as e:
logger.error(f"请求失败: {e}")
if i < retry_times - 1:
time.sleep(random.uniform(3, 5))
else:
return None

return None

def parse_list_page(self, html, base_url):
"""解析列表页，提取详情页URL和翻页链接"""
if not html:
return [], None

soup = BeautifulSoup(html, 'html.parser')
detail_urls = []

常见的选择器模式（可以根据实际情况修改）

selectors = [
'a[href="detail"]', # 包含detail的链接
'a[href="info"]', # 包含info的链接
'a[href*="content"]', # 包含content的链接
'.news-title a', # class为news-title下的链接
'.list-item a', # class为list-item下的链接
'h3 a', # h3标签下的链接
'td a', # 表格中的链接
]

for selector in selectors:
links = soup.select(selector)
for link in links:
href = link.get('href')
if href and href not in ['#', 'javascript:void(0)']:
full_url = urljoin(base_url, href)
if full_url not in detail_urls:
detail_urls.append(full_url)

查找下一页链接

next_page = None
next_selectors = [
'a:contains("下一页")',
'a:contains("下页")',
'a.next',
'.next a',
'a[rel="next"]',
'.pagination .next',
]

for selector in next_selectors:
next_link = soup.select_one(selector)
if next_link and next_link.get('href'):
next_page = urljoin(base_url, next_link['href'])
break

return detail_urls, next_page

def parse_detail_page(self, html, url):
"""解析详情页，提取具体内容"""
if not html:
return {}

soup = BeautifulSoup(html, 'html.parser')

移除脚本和样式

for script in soup(["script", "style"]):
script.decompose()

提取标题（常见选择器）

title = None
title_selectors = [
'h1',
'.title',
'.article-title',
'.news-title',
'h2',
'h3',
]

for selector in title_selectors:
title_elem = soup.select_one(selector)
if title_elem:
title = title_elem.get_text().strip()
break

提取时间

date = None
date_selectors = [
'.time',
'.date',
'.publish-time',
'time',
'.article-time',
]

for selector in date_selectors:
date_elem = soup.select_one(selector)
if date_elem:
date = date_elem.get_text().strip()
break

提取正文内容

content = None
content_selectors = [
'.content',
'.article-content',
'.news-content',
'.detail-content',
'.main-text',
'#content',
'article',
]

for selector in content_selectors:
content_elem = soup.select_one(selector)
if content_elem:
content = content_elem.get_text().strip()
break

如果没有找到特定容器，获取body文本

if not content:
body = soup.find('body')
if body:
content = body.get_text().strip()

return {
'url': url,
'title': title,
'date': date,
'content': content,
'crawl_time': time.strftime('%Y-%m-%d %H:%M:%S')
}

def save_to_csv(self, data, filename='data.csv'):
"""保存数据到CSV"""
filepath = os.path.join(self.data_dir, filename)

file_exists = os.path.isfile(filepath)

with open(filepath, 'a', newline='', encoding='utf-8-sig') as f:
writer = csv.DictWriter(f, fieldnames=['url', 'title', 'date', 'content', 'crawl_time'])

if not file_exists:
writer.writeheader()

writer.writerow(data)

logger.info(f"数据已保存到 {filepath}")

def save_to_txt(self, data, filename='data.txt'):
"""保存数据到TXT"""
filepath = os.path.join(self.data_dir, filename)

with open(filepath, 'a', encoding='utf-8') as f:
f.write(f"URL: {data['url']}\n")
f.write(f"标题: {data['title']}\n")
f.write(f"日期: {data['date']}\n")
f.write(f"内容:\n{data['content']}\n")
f.write("="*50 + "\n\n")

def crawl(self, start_url, max_pages=10, delay=2):
"""主爬取流程"""
current_url = start_url
page_count = 0
detail_count = 0

while current_url and page_count < max_pages:
logger.info(f"正在爬取列表页 {page_count + 1}: {current_url}")

获取列表页

list_html = self.fetch_page(current_url)
if not list_html:
break

解析列表页

detail_urls, next_page = self.parse_list_page(list_html, current_url)

爬取详情页

for detail_url in detail_urls:
logger.info(f"正在爬取详情页 {detail_count + 1}: {detail_url}")

获取详情页

detail_html = self.fetch_page(detail_url)
if not detail_html:
continue

解析详情页

data = self.parse_detail_page(detail_html, detail_url)

保存数据

if data and data['content']:
self.save_to_csv(data)
# 也可以同时保存为txt
# self.save_to_txt(data)
detail_count += 1

随机延时

time.sleep(random.uniform(delay, delay + 2))

翻页

current_url = next_page
page_count += 1

翻页延时

time.sleep(random.uniform(delay * 2, delay * 3))

logger.info(f"爬取完成！共爬取 {page_count} 页，{detail_count} 条详情")

================= 使用示例 =================

def main():
# 创建爬虫实例
spider = GenericSpider()

示例1：爬取普通新闻网站

spider.crawl(
start_url='http://example.com/news/page/1/',
max_pages=5, # 最大爬取页数
delay=2 # 延时秒数
)

示例2：针对特定网站定制解析规则

可以继承GenericSpider并重写parse_list_page和parse_detail_page方法

"""
class CustomSpider(GenericSpider):
def parse_list_page(self, html, base_url):
soup = BeautifulSoup(html, 'html.parser')
detail_urls = []

定制化的解析逻辑

for item in soup.select('.custom-list .item a'):
href = item.get('href')
if href:
detail_urls.append(urljoin(base_url, href))

定制化的翻页逻辑

next_page = None
next_link = soup.select_one('.custom-pagination .next')
if next_link and next_link.get('href'):
next_page = urljoin(base_url, next_link['href'])