多进程和多线程爬虫案例
import os.path
import time
from multiprocessing import Process
from threading import Thread
import requests
from lxml import etree
from fake_useragent import UserAgent
class BaseSpider(object):
def __init__(self):
self.url_list = self.create_url_list()
# self.url_list = ['https://pic.netbian.com/4kdongman/']
self.headers = {
'User-Agent': UserAgent().random
}
self.BASE_DIR = os.path.dirname(__file__)
self.file_name_path = self.create_file_name()
# 创建url列表
def create_url_list(self):
url_list = []
for i in range(1, 10):
if i == 1:
index_url = 'https://pic.netbian.com/4kdongman/'
url_list.append(index_url)
else:
index_url = f'https://pic.netbian.com/4kdongman/index_{i}.html'
url_list.append(index_url)
return url_list
def get_tree(self, page_text):
tree = etree.HTML(page_text)
return tree
def get_page_text(self, url, encoding='gbk'):
response = requests.get(url, headers=self.headers)
response.encoding = encoding
return response.text
def create_file_name(self, path='img'):
file_name_path = os.path.join(self.BASE_DIR, path)
os.makedirs(file_name_path, exist_ok=True)
return file_name_path
class SpiderImg(BaseSpider):
def __init__(self):
super().__init__()
@staticmethod
def timer(func):
def inner(*args, **kwargs):
start_time = time.time()
res = func(*args, **kwargs)
print(f" {func.__name__} | 总耗时 :>>>> {time.time() - start_time} s")
return res
return inner
def spider_index_tree(self):
tree_list = []
for url in self.url_list:
# 获取每一页的页面源码
page_text = self.get_page_text(url=url)
tree = self.get_tree(page_text=page_text)
tree_list.append(tree)
return tree_list
def __get_tree_data(self, tree):
img_data_list = []
li_list = tree.xpath('//*[@id="main"]/div[3]/ul/li')
for li in li_list:
# ./a/img
img_title = li.xpath('./a/img/@alt')[0]
img_src = 'https://pic.netbian.com' + li.xpath('./a/img/@src')[0]
img_data_list.append({'img_title': img_title, 'img_src': img_src})
return img_data_list
def spider_index_img_data(self):
img_data_list = []
tree_list = self.spider_index_tree()
for tree in tree_list:
img_list = self.__get_tree_data(tree=tree)
# [{},{}]
img_data_list.extend(img_list)
return img_data_list
def download(self, img_src, img_title):
response = requests.get(url=img_src)
file_path = os.path.join(self.file_name_path, f'{img_title}.png')
with open(file_path, mode='wb') as fp:
for data in response.iter_content():
fp.write(data)
print(f"当前图片 :>>>> {img_title} 保存成功!")
@timer
def download_normal(self):
img_data_list = self.spider_index_img_data()
for img_data in img_data_list:
img_title = img_data.get('img_title')
img_src = img_data.get('img_src')
self.download(img_src=img_src, img_title=img_title)
@timer
def download_process(self):
img_data_list = self.spider_index_img_data()
task_list = [Process(target=self.download, args=(img_data.get('img_src'), img_data.get('img_title'))) for
img_data in img_data_list]
[task.start() for task in task_list]
[task.join() for task in task_list]
@timer
def download_thread(self):
img_data_list = self.spider_index_img_data()
task_list = [Thread(target=self.download, args=(img_data.get('img_src'), img_data.get('img_title'))) for
img_data in img_data_list]
[task.start() for task in task_list]
[task.join() for task in task_list]
if __name__ == '__main__':
spider = SpiderImg()
# spider.download_normal() # download_normal | 总耗时 :>>>> 31.3393292427063 s
# spider.download_process() # download_process | 总耗时 :>>>> 34.51722550392151 s
spider.download_thread() # download_thread | 总耗时 :>>>> 15.272460699081421 s
'''
# num_list_one = [1, 2, 3, 4]
# num_list_two = [7, 8, 9, 10]
# num_list_new = []
# print(num_list_new)
# num_list_new.extend(num_list_one)
# num_list_new.extend(num_list_two)
# print(num_list_new)
'''
协程案例
import asyncio
import os
import time
from fake_useragent import UserAgent
import aiohttp
from lxml import etree
headers = {
'User-Agent': UserAgent().random
}
BASE_DIR = os.path.dirname(__file__)
def create_file_name(path='img'):
file_name_path = os.path.join(BASE_DIR, path)
os.makedirs(file_name_path, exist_ok=True)
return file_name_path
file_name_path = create_file_name()
async def create_url_list():
url_list = []
for i in range(1, 10):
if i == 1:
index_url = 'https://pic.netbian.com/4kdongman/'
url_list.append(index_url)
else:
index_url = f'https://pic.netbian.com/4kdongman/index_{i}.html'
url_list.append(index_url)
return url_list
async def get_tree(page_text):
tree = etree.HTML(page_text)
return tree
async def get_page_text(tag_url, encoding='gbk'):
async with aiohttp.ClientSession() as session:
# 如果遇到 ssl error 这种错，一般都是 ssl=False
async with session.get(url=tag_url, headers=headers, ssl=False) as response:
page_text = await response.text(encoding='gbk')
return page_text
async def spider_index_tree():
tree_list = []
url_list = await create_url_list()
# url_list = ['https://pic.netbian.com/4kdongman/']
for url in url_list:
# 获取每一页的页面源码
page_text = await get_page_text(tag_url=url)
tree = await get_tree(page_text=page_text)
tree_list.append(tree)
return tree_list
async def get_tree_data(tree):
img_data_list = []
li_list = tree.xpath('//*[@id="main"]/div[3]/ul/li')
for li in li_list:
# ./a/img
img_title = li.xpath('./a/img/@alt')[0]
img_src = 'https://pic.netbian.com' + li.xpath('./a/img/@src')[0]
img_data_list.append({'img_title': img_title, 'img_src': img_src})
return img_data_list
async def spider_index_img_data():
img_data_list = []
tree_list = await spider_index_tree()
for tree in tree_list:
img_list = await get_tree_data(tree=tree)
# [{},{}]
img_data_list.extend(img_list)
return img_data_list
async def download(img_src, img_title):
async with aiohttp.ClientSession() as session:
async with session.get(url=img_src, headers=headers, ssl=False) as response:
data_all = await response.read()
file_path = os.path.join(file_name_path, f'{img_title}.png')
with open(file_path, mode='wb') as fp:
fp.write(data_all)
print(f"当前图片 :>>>> {img_title} 保存成功!")
async def main():
img_data_list = await spider_index_img_data()
# 创建Task对象列表
task_list = [asyncio.create_task(download(img_src=img_data.get('img_src'), img_title=img_data.get('img_title'))) for
img_data in img_data_list]
# 等待任务完成
await asyncio.wait(task_list)
if __name__ == '__main__':
start_time = time.time()
# 启协程
asyncio.run(main())
print(f"总耗时 :>>>> {time.time() - start_time} s")
# 总耗时 :>>>> 6.5860209465026855 s
