python 多进程并行密集型计算,单参数于多参数的实例
import multiprocessing
import time
import numba as nb
class parallelPool:
def __init__(self,cpu_count:int=multiprocessing.cpu_count()):
self.pool = multiprocessing.Pool(cpu_count)
self.results = []
self.taskList = []
def submit(self,funcItem:tuple or list):
# print(type(funcItem))
if type(funcItem) == tuple:
self._run(funcItem)
elif type(funcItem) == list:
for item in funcItem:
self._run(item)
def _run(self,item:tuple):
arg = []
for i in range(1, item.__len__()):
arg.append(item[i])
self.taskList.append(self.pool.apply_async(item[0], args=tuple(arg)))
def execute(self):
for res in self.taskList:
self.results.append(res.get())
# # 获取结果
self.pool.close()
self.pool.join()
return self.results
@nb.jit()
def _fbnq(n: int):
"""实际计算函数"""
if n < 1:
return 1
else:
return _fbnq(n - 1) + _fbnq(n - 2)
def task1(n):
return _fbnq(n)
def task2(n):
return _fbnq(n)
def task3(n,p,g):
return _fbnq(n) + p + g
if __name__ == '__main__':
# 创建进程池(使用所有CPU核心)
st = time.time()
# 准备不同任务列表
tasks = [
(task1, 40), (task2, 40), (task3, 40,1,1),
(task1, 40), (task2, 40), (task3, 40,1,1),
(task1, 40), (task2, 40), (task3, 40, 1,3),
(task1, 40), (task2, 40), (task3, 40, 1,4),
(task1, 40), (task2, 40), (task3, 40, 1,5),
(task1, 40), (task2, 40), (task3, 40, 1,6)
]*3
# 并行执行任务
pl = parallelPool()
# for item in tasks: # 9.326000213623047 9.46895694732666 23.66971516609192
# pl.submit(item)
pl.submit((task3,41,1,1))
pl.submit(tasks) # 9.486266851425171 9.4131498336792 23.88749670982361
rlts = pl.execute()
print(rlts)
print(rlts.__len__())
print(time.time()-st)
自动化学习。

浙公网安备 33010602011771号