python threading 多线程、 ThreadPoolExecutor 线程池 及 ProcessPoolExecutor 进程池 测试用例

1、通过线程池 ThreadPoolExecutor 方式实现多线程，同时控制并发数（ProcessPoolExecutor进程池同理） 

线程池的基类是 concurrent.futures 模块中的 Executor，Executor 提供了两个子类，即 ThreadPoolExecutor 和 ProcessPoolExecutor，其中 ThreadPoolExecutor 用于创建线程池，而 ProcessPoolExecutor 用于创建进程池。

只要涉及到需要使用多线程或者多进程处理，强烈推荐线程池或者进程池方式处理。

# coding:utf-8
from concurrent.futures import ThreadPoolExecutor
import time


def do_work(num):
    print("Number {} is coming.".format(num))
    time.sleep(num)
    print("Number {} is done.".format(num))
    return "The result of the thread {}".format(str(num))


def do_main():
    do_list = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
    # 创建线程池，线程池并发数6
    # thread_pool = ThreadPoolExecutor(max_workers=6)

    # 将需要执行的task任务提交给线程池，submit为每个线程返回一个对象
    # for do_item in do_list:
    #     future = thread_pool.submit(do_work, do_item)

    # 或者将可迭代对象传给map处理，map为每个可迭代元素启动一个线程
    # results = thread_pool.map(do_work, do_list)

    # 关闭线程池
    # thread_pool.shutdown(wait=True)

    # 最简便方式，使用with启用上下文管理，集创建线程池、提交task任务、关闭线程池于一体
    with ThreadPoolExecutor(max_workers=6) as thread_pool:
        results = thread_pool.map(do_work, do_list)

    # 获取各个线程执行的结果
    for result in results:
        print(result)


if __name__ == '__main__':
    do_main()

 

2、通过Tread类创建线程，同时控制并发数量

# coding:utf-8
import threading
import time


# 多线程执行某个作业，控制同时执行的作业数量,此次测试控制在5个线程内
do_list = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
thread_queue = 5
thread_list = []


def do_work(num):
    print("Number {} is coming.".format(num))
    time.sleep(num)
    print("Number {} is done.".format(num))


def do_main():
    for do_item in do_list:
        # 获取正在执行的线程数，小于控制队列，则继续加入新的线程作业，
        while True:
            running_cnt = len(list(filter(lambda item: item.isAlive(), thread_list)))
            if running_cnt < thread_queue:
                thread_item = threading.Thread(target=do_work, args=(do_item,))
                thread_list.append(thread_item)
                thread_item.start()
                break
            else:
                time.sleep(0.01)

    # 检测是否还有正在执行的线程，等待所有线程执行完毕后，结束主线程
    # join的原理就是依次检验线程池中的线程是否结束，没有结束就阻塞直到线程结束，如果结束则跳转执行下一个线程的join函数
    for thr in thread_list:
        thr.join()

    # 另一种检测是否还有正在执行的线程的方式，等待所有线程执行完毕后，结束主线程
    # running_check_cnt = len(list(filter(lambda item: item.isAlive(), thread_list)))
    # while running_check_cnt > 0:
    #     running_check_cnt = len(list(filter(lambda item: item.isAlive(), thread_list)))
    #     time.sleep(0.01)

    print("所有线程已执行完毕")


if __name__ == '__main__':
    do_main()

 

3、重写Thread类，通过创建新类对象实现多线程

# coding:utf-8
from threading import Thread
import time


# 重写多线程类
class WorkThread(Thread):
    def __init__(self, num):
        Thread.__init__(self)
        self.number = num

    def run(self):
        print("Number {} is coming.".format(self.number))
        time.sleep(self.number)
        print("Number {} is done.".format(self.number))


def do_main():
    do_list = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
    thread_list = []

    for do_item in do_list:
        thread_item = WorkThread(do_item)
        thread_list.append(thread_item)
        thread_item.start()

    # 检测是否还有正在执行的线程，等待所有线程执行完毕后，结束主线程
    running_check_cnt = len(list(filter(lambda item: item.isAlive(), thread_list)))
    while running_check_cnt > 0:
        running_check_cnt = len(list(filter(lambda item: item.isAlive(), thread_list)))
        time.sleep(0.01)

    print("所有线程已执行完毕")


if __name__ == '__main__':
    do_main()