python-线程池-进程池

线程池进程池 concurrent.futures

• 什么是池
  • 要在程序开始的时候，还没提交任务先创建几个线程或者进程放在一个池子里，这就是池
• 为什么要用池
  • 如果先开好进程/线程，那么有任务之后就可以直接使用这个池中的数据了
  • 并且开好的线程或者进程会一直存在在池中，可以被多个任务反复利用。这样极大的减少了开启\关闭\调度线程/进程的时间开销
  • 池中的线程/进程个数控制了操作系统需要调度的任务个数，控制池中的单位，有利于跳操作系统的效率，减轻操作系统的负担

---

concurrent.futures模块

from concurrent.futures import ThreadPoolExecutor,ProcessPoolExecutor

ThreadPoolExecutor线程池 ProcessPoolExecutor进程池

• 实例化创建池 和 向池中提交任务：submit
• 线程池

from concurrent.futures import ThreadPoolExecutor,ProcessPoolExecutor

from threading import current_thread


def func():
    print(current_thread().ident)

t_pool = ThreadPoolExecutor(4)  # 线程池中线程的数量为4

for i in range(20):
    t_pool.submit(func)  # submit异步提交任务

import time, random
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
from threading import current_thread


def func():
    print(current_thread().ident, 'start')
    time.sleep(random.randint(1, 4))
    print(current_thread().ident, 'end')


t_pool = ThreadPoolExecutor(4)  # 线程池中线程的数量为4

for i in range(20):
    t_pool.submit(func)  # submit异步提交任务

• 传参数（可以位置传参，关键字传参）

import time, random
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
from threading import current_thread


def func(name,name2):
    print(current_thread().ident, name)
    time.sleep(random.randint(1, 4))
    print(current_thread().ident, name2)


t_pool = ThreadPoolExecutor(4)  # 线程池中线程的数量为4

for i in range(20):
    t_pool.submit(func, 'alex', 'taibai')  # submit异步提交任务

• 开启进程池（进程池不要忘了加if __name__ = '__main__':）

import time, random
import os
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor


def func(name,name2):
    print(os.getpid(), name)  # 当前进程pid
    time.sleep(random.randint(1, 4))
    print(os.getpid(), name2)

if __name__ == '__main__':  # 记得加
    t_pool = ProcessPoolExecutor(4)  # 进程池中进程的数量为4

    for i in range(20):
        t_pool.submit(func, 'alex', 'taibai')  # submit异步提交任务

• 获取任务结果

from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
import time
import os
import random


def func(a, b):
    print(os.getpid(), 'start', a, b)
    time.sleep(random.randint(1, 4))
    print(os.getpid(), 'end')
    return a+b


if __name__ == '__main__':
    tp = ProcessPoolExecutor(4)
    future_l = {}
    for i in range(20):  # 异步非阻塞的
        ret = tp.submit(func, i, i+1)
        future_l[i] = ret
        #  print(ret.result())  # 变成串行了
    for key in future_l:  # 同步阻塞的
        print(key, future_l[key].result())

• map（只适合传递简单的参数，并且必须是一个可迭代的数据类型作为参数）

# 效果和上面获取任务结果一样，只是少了个for循环
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
import time
import os
import random


def func(a):
    b = a+1
    print(os.getpid(), 'start', a, b)
    time.sleep(random.randint(1, 4))
    print(os.getpid(), 'end')
    return a+b


if __name__ == '__main__':
    tp = ProcessPoolExecutor(4)
    ret = tp.map(func, range(20))
    for key in ret:  # 同步阻塞
        print(key)

• 回调函数 ret.add_done_callback(函数) （效率最高）

from concurrent.futures import ThreadPoolExecutor
from threading import current_thread
import time
import random


def func(a, b):
    print(current_thread().ident, 'start', a, b)
    time.sleep(random.randint(1, 4))
    print(current_thread().ident, 'end')
    return a+b  # 可以做个标识  return (a, a+b)

def print_func(ret):  # 异步阻塞
    print(ret.result())


if __name__ == '__main__':
    tp = ThreadPoolExecutor(4)  # 开四个线程
    for i in range(20):  # 异步非阻塞
        ret = tp.submit(func, i, i+1)  # 往线程池中提交任务
        ret.add_done_callback(print_func)  # 回调函数
        # ret这个任务再执行完毕的瞬间立即触发print_func函数，并且把任务的返回值对象传递到print_func做参数
        # 异步阻塞 回调函数 给ret对象绑定一个回调函数，等待ret对应的任务有了结果之后立即调用print_func这个函数，就可以对结果立即进行处理，而不用按照顺序接收处理结果。。异步阻塞

• shutdown

from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
import time
import os
import random


def func(a, b):
    print(os.getpid(), 'start', a, b)
    time.sleep(random.randint(1, 4))
    print(os.getpid(), 'end')
    return a+b


if __name__ == '__main__':
    tp = ProcessPoolExecutor(4)
    future_l = {}
    for i in range(20):  # 异步非阻塞的
        ret = tp.submit(func, i, i+1)
        future_l[i] = ret
    tp.shutdown()  # 关闭线程池，等待线程池中所有的任务运行完毕
        #  print(ret.result())  # 变成串行了
    for key in future_l:  # 同步阻塞的
        print(key, future_l[key].result())