python ThreadPoolExecutor使用后内存不断消耗内存解决

from functools import wraps
import concurrent.futures
import time


def test_func(func):
    @wraps(func)
    def inner(*args, **kwargs):
        print ("start...")
        res = func(*args, **kwargs)
        print ("end...")
        return res
    return inner

@test_func
def test1():
    print ('run test1 ....')
    time.sleep(10)

    return 1

@test_func
def test2():
    print ('run test2 ....')
    time.sleep(10)

    return 1

@test_func
def test3():
    print ('run test3 ....')
    time.sleep(10)

    return 1

@test_func
def test4():
    print ('run test4 ....')
    time.sleep(10)

    return 1



def main():

    funcs = [test1, test2, test3, test4]

    executor = concurrent.futures.ThreadPoolExecutor(4)
     while True:
             for func in funcs:
                    executor.submit(func)
             print executor._work_queue.qsize()

 

通过运行代码能发现在调用该程序之后，内存直线上升；在循环调用线程池时，进程会不断的往线程池中扔任务，而不会判断，等待线程池中是否存在空闲线程程；

解决方法

既然线程池使用的为无界队列，那么就可以将类重写，并使用有界队列，如：

import queue
from concurrent.futures import ThreadPoolExecutor

class BoundThreadPollExecutor(ThreadPoolExecutor):
    def __init__(self, *args, **kwargs):
        super(BoundThreadPollExecutor, self).__init__(*args, **kwargs)
        self._work_queue = queue.Queue(4)